Sync from GitHub via hub-sync

evals/run_eval.py

@@ -76,9 +76,19 @@ def matches_expected(actual_path: str, expected_sources) -> bool:
 
 
 def compute_retrieval_metrics(expected_sources, actual_sources):
-    hit = any(matches_expected(path, expected_sources) for path in actual_sources)
-    top1 = bool(actual_sources) and matches_expected(actual_sources[0], expected_sources)
-    return {"retrieval_hit": int(hit), "top1_hit": int(top1)}
+    matching_ranks = [
+        rank
+        for rank, path in enumerate(actual_sources, start=1)
+        if matches_expected(path, expected_sources)
+    ]
+    hit = bool(matching_ranks)
+    top1 = bool(matching_ranks and matching_ranks[0] == 1)
+    reciprocal_rank = 1.0 / matching_ranks[0] if matching_ranks else 0.0
+    return {
+        "retrieval_hit": int(hit),
+        "top1_hit": int(top1),
+        "reciprocal_rank": reciprocal_rank,
+    }
 
 
 def keyword_hits(answer: str, keywords):
@@ -97,7 +107,7 @@ def judge_faithfulness(rag_system, question: str, answer: str, sources: list):
     if not ENABLE_FAITHFULNESS or not answer.strip() or not sources:
         return None
     context = "\n\n".join(
-        f"[{i}] {source['file_path']}\n{source['snippet'][:800]}"
+        f"[{i}] {source['file_path']}\n{source['snippet'][:1500]}"
         for i, source in enumerate(sources, start=1)
     )
     system_prompt = (
@@ -141,6 +151,7 @@ def run_case(rag_system, repo_id: int, repo_name: str, case: dict):
         question=case["question"],
         top_k=TOP_K,
         history=case.get("turns", []),
+        debug_retrieval=True,
     )
     elapsed_ms = (time.time() - start) * 1000
 
@@ -149,7 +160,18 @@ def run_case(rag_system, repo_id: int, repo_name: str, case: dict):
     retrieval = compute_retrieval_metrics(case.get("expected_sources", []), cited_paths)
     matched, total_keywords = keyword_hits(result.get("answer", ""), case.get("must_include_any", []))
     has_citations = bool(result.get("citations"))
-    grounded = retrieval["retrieval_hit"] == 1 and has_citations and (total_keywords == 0 or matched > 0)
+    expected_source_grounded = (
+        retrieval["retrieval_hit"] == 1
+        and has_citations
+        and (total_keywords == 0 or matched > 0)
+    )
+
+    retrieval_debug = result.get("retrieval_debug", [])
+    for item in retrieval_debug:
+        item["expected_source"] = matches_expected(
+            item.get("file_path", ""),
+            case.get("expected_sources", []),
+        )
 
     return {
         "id": case.get("id", case["question"]),
@@ -162,7 +184,11 @@ def run_case(rag_system, repo_id: int, repo_name: str, case: dict):
         "retrieved_sources": cited_paths,
         "retrieval_hit": retrieval["retrieval_hit"],
         "top1_hit": retrieval["top1_hit"],
-        "grounded": int(grounded),
+        "reciprocal_rank": round(retrieval["reciprocal_rank"], 4),
+        "expected_source_grounded": int(expected_source_grounded),
+        # Backward-compatible alias for older report consumers.
+        "grounded": int(expected_source_grounded),
+        "retrieval_debug": retrieval_debug,
         "faithfulness": judge_faithfulness(rag_system, case["question"], result.get("answer", ""), sources),
         "latency_ms": round(elapsed_ms, 1),
     }
@@ -178,6 +204,11 @@ def summarize(details):
         "case_count": len(details),
         "retrieval_hit_rate": round(mean(item["retrieval_hit"] for item in details), 4),
         "top1_hit_rate": round(mean(item["top1_hit"] for item in details), 4),
+        "mrr": round(mean(item["reciprocal_rank"] for item in details), 4),
+        "expected_source_grounded_rate": round(
+            mean(item["expected_source_grounded"] for item in details), 4
+        ),
+        # Backward-compatible alias.
         "grounded_answer_rate": round(mean(item["grounded"] for item in details), 4),
         "faithfulness": round(mean(faith_scores), 4) if faith_scores else None,
         "latency_p95_ms": round(latencies[p95_index], 1),

evals/sample_eval_set.json

@@ -17,31 +17,31 @@
           "id": "documenso-api-v2-document-router",
           "category": "api",
           "question": "Where is the current document API implemented and how is it exposed?",
-          "ground_truth": "The current API V2 is implemented under packages/trpc/server, with document operations organized under a document router. It is exposed through /api/v2 and /api/v2-beta with tRPC and OpenAPI support, and it accepts API-token or session-cookie authentication depending on the route.",
+          "ground_truth": "The current API V2 is implemented under packages/trpc/server, with document operations organized under a document router. It is exposed through API V2 routes with tRPC and OpenAPI support.",
           "expected_sources": ["ARCHITECTURE.md", "packages/trpc/server", "packages/trpc/server/document-router", "apps/remix/server"],
           "must_include_any": ["packages/trpc/server", "document-router", "API V2", "OpenAPI", "tRPC"]
         },
         {
           "id": "documenso-signing-package",
           "category": "specific-function",
-          "question": "What does the signing package do in Documenso?",
+          "question": "What does the @documenso/signing package do, and where is it implemented?",
           "ground_truth": "The @documenso/signing package owns PDF signing behavior. Its signPdf entry point selects a signing transport, applies timestamp authority settings when configured, and supports local P12 signing and Google Cloud KMS/HSM-backed signing through transport implementations.",
           "expected_sources": ["ARCHITECTURE.md", "packages/signing/index.ts", "packages/signing/helpers", "packages/signing/transports", ".env.example"],
-          "must_include_any": ["PDF signing", "transports", "local", "Google", "KMS"]
+          "must_include_any": ["PDF signing", "packages/signing", "transports", "local", "Google", "KMS"]
         },
         {
           "id": "documenso-document-send-flow",
           "category": "cross-file",
           "question": "How does a document send operation flow across the Documenso codebase?",
-          "ground_truth": "A document send operation starts at an API or UI route, goes through the API layer such as packages/trpc/server/document-router, delegates core behavior to packages/lib/server-only/document and related recipient or field logic, persists through packages/prisma, and can trigger emails or jobs through packages/email and packages/lib/jobs.",
+          "ground_truth": "A document send operation starts at an API or UI route, delegates core behavior to server-only document logic and related recipient or field logic, persists state through the data layer, and can trigger emails or jobs.",
           "expected_sources": ["packages/trpc/server/document-router", "packages/lib/server-only/document", "packages/lib/server-only/recipient", "packages/prisma", "packages/email", "packages/lib/jobs"],
-          "must_include_any": ["document-router", "server-only/document", "prisma", "email", "jobs"]
+          "must_include_any": ["document-router", "server-only/document", "recipient", "prisma", "email", "jobs"]
         },
         {
           "id": "documenso-required-env",
           "category": "config-setup",
           "question": "Which environment variables are central to running a self-hosted Documenso instance?",
-          "ground_truth": "The setup expects values such as NEXTAUTH_SECRET, NEXT_PRIVATE_ENCRYPTION_KEY, NEXT_PRIVATE_ENCRYPTION_SECONDARY_KEY, NEXT_PUBLIC_WEBAPP_URL, NEXT_PRIVATE_DATABASE_URL, NEXT_PRIVATE_DIRECT_DATABASE_URL, NEXT_PRIVATE_SMTP_FROM_NAME, and NEXT_PRIVATE_SMTP_FROM_ADDRESS. The env example and self-hosting docs also cover internal URLs, signing transport, storage transport, and optional OAuth and webhook configuration.",
+          "ground_truth": "Self-hosting requires configuration for authentication, encryption, the public web URL, database connectivity, and SMTP/email delivery. The environment example and self-hosting documentation are the primary sources.",
           "expected_sources": ["README.md", ".env.example", "apps/docs/content/docs/self-hosting/configuration/environment.mdx"],
           "must_include_any": ["NEXTAUTH_SECRET", "NEXT_PUBLIC_WEBAPP_URL", "DATABASE_URL", "SMTP", "encryption"]
         },
@@ -49,7 +49,7 @@
           "id": "documenso-playwright-tests",
           "category": "tests",
           "question": "Where does Documenso keep end-to-end app tests?",
-          "ground_truth": "The architecture identifies @documenso/app-tests as the E2E test package, and the packages/app-tests directory is intended for Playwright coverage of app behavior.",
+          "ground_truth": "Documenso keeps end-to-end application tests in the @documenso/app-tests package under packages/app-tests, using Playwright for app-level coverage.",
           "expected_sources": ["ARCHITECTURE.md", "packages/app-tests", "packages/app-tests/package.json"],
           "must_include_any": ["app-tests", "E2E", "Playwright"]
         },
@@ -57,7 +57,7 @@
           "id": "documenso-webhook-security-errors",
           "category": "error-handling",
           "question": "Where should you look for webhook security or SSRF-related safeguards?",
-          "ground_truth": "Webhook safeguards belong in packages/lib/server-only/webhooks, with related configuration documented in .env.example such as NEXT_PRIVATE_WEBHOOK_SSRF_BYPASS_HOSTS. These sources explain where outbound webhook behavior and security exceptions are controlled.",
+          "ground_truth": "Webhook security safeguards are implemented in packages/lib/server-only/webhooks, with related configuration such as SSRF bypass hosts documented in environment configuration.",
           "expected_sources": [".env.example", "packages/lib/server-only/webhooks"],
           "must_include_any": ["webhook", "SSRF", "bypass", "hosts"]
         },
@@ -66,87 +66,12 @@
           "category": "conversation",
           "turns": [
             {"role": "user", "content": "How does Documenso seal or sign completed documents?"},
-            {"role": "assistant", "content": "PDF completion crosses server-only PDF/document logic and the @documenso/signing package, which supports local and Google-backed signing transports."}
+            {"role": "assistant", "content": "Completed PDFs cross server-only document/job logic and the @documenso/signing package, whose signPdf entry point supports multiple signing transports."}
           ],
-          "question": "show me where signing plugs in",
-          "ground_truth": "The follow-up should retrieve packages/signing, especially the signPdf entry point, transports, and helpers, plus the seal-document job where completed PDFs are decorated and passed into signing, and signing-related env configuration.",
+          "question": "show me where that signing package plugs in",
+          "ground_truth": "The follow-up should connect the @documenso/signing package, especially its entry point, transports, and helpers, with the seal-document job that passes completed PDFs into signing and with relevant signing configuration.",
           "expected_sources": ["packages/signing/index.ts", "packages/signing/transports", "packages/signing/helpers", "packages/lib/jobs/definitions/internal/seal-document.handler.ts", ".env.example"],
-          "must_include_any": ["packages/signing", "transports", "PDF", "signing"]
-        }
-      ]
-    },
-    {
-      "id": "sqlite",
-      "name": "SQLite",
-      "github_url": "https://github.com/sqlite/sqlite.git",
-      "cases": [
-        {
-          "id": "sqlite-purpose",
-          "category": "architecture",
-          "question": "What is SQLite and how is the project organized at a high level?",
-          "ground_truth": "SQLite is a self-contained, serverless, zero-configuration embedded SQL database engine. Its core sources live under src/, tests are TCL scripts, the build produces both a library and a single-file amalgamation, and the project's own AGENTS.md documents the execution pipeline for contributors.",
-          "expected_sources": ["README.md", "src", "AGENTS.md"],
-          "must_include_any": ["embedded", "serverless", "amalgamation", "SQL database engine"]
-        },
-        {
-          "id": "sqlite-query-pipeline",
-          "category": "cross-file",
-          "question": "How does a SQL statement flow through SQLite's execution pipeline?",
-          "ground_truth": "SQL text is tokenized in tokenize.c, parsed by the Lemon-generated parser from parse.y, turned into VDBE bytecode by the code generator files (build.c, select.c, insert.c, update.c, delete.c, expr.c) after being optimized by the where*.c query optimizer, then executed by the VDBE in vdbe.c against the B-tree layer in btree.c, the pager in pager.c, the WAL in wal.c, and finally the OS-level VFS such as os_unix.c.",
-          "expected_sources": ["src/tokenize.c", "src/parse.y", "src/vdbe.c", "src/btree.c", "src/pager.c", "src/wal.c", "AGENTS.md"],
-          "must_include_any": ["tokenizer", "parser", "VDBE", "B-Tree", "pager", "WAL"]
-        },
-        {
-          "id": "sqlite-btree-role",
-          "category": "specific-function",
-          "question": "What role does btree.c play in SQLite?",
-          "ground_truth": "btree.c implements the B-tree storage engine SQLite uses to organize table and index data on disk. Its public interface is declared in btree.h, while btreeInt.h defines the data structures used only internally by the module.",
-          "expected_sources": ["src/btree.c", "src/btree.h", "src/btreeInt.h"],
-          "must_include_any": ["B-Tree", "storage engine", "btree.h"]
-        },
-        {
-          "id": "sqlite-vdbe-opcodes",
-          "category": "specific-function",
-          "question": "How are VDBE opcodes generated and where do they come from?",
-          "ground_truth": "VDBE opcode numbers and names are extracted automatically by scanning src/vdbe.c with the mkopcodeh.tcl script, which generates opcodes.h; a second script, mkopcodec.tcl, then generates opcodes.c, which provides the reverse opcode-to-name mapping used for EXPLAIN output.",
-          "expected_sources": ["src/vdbe.c", "mkopcodeh.tcl", "mkopcodec.tcl"],
-          "must_include_any": ["opcodes.h", "mkopcodeh.tcl", "VDBE", "EXPLAIN"]
-        },
-        {
-          "id": "sqlite-parser-generation",
-          "category": "config-setup",
-          "question": "How is the SQL grammar parser built for SQLite?",
-          "ground_truth": "The grammar is defined in src/parse.y and compiled into parse.c by the Lemon LALR(1) parser generator in tool/lemon.c, which uses tool/lempar.c as a template and also emits the parse.h header as a side effect.",
-          "expected_sources": ["src/parse.y", "tool/lemon.c", "tool/lempar.c"],
-          "must_include_any": ["parse.y", "Lemon", "LALR", "parse.c"]
-        },
-        {
-          "id": "sqlite-testing",
-          "category": "tests",
-          "question": "How does SQLite run its test suite and what kind of tests does it use?",
-          "ground_truth": "SQLite's tests are TCL scripts executed through the testfixture binary, which is built with make testfixture. AGENTS.md instructs contributors to run at least make devtest after any change under src/, and make sqlite3.c builds the amalgamation used for distribution.",
-          "expected_sources": ["test", "AGENTS.md", "Makefile.in"],
-          "must_include_any": ["testfixture", "TCL", "devtest", "test suite"]
-        },
-        {
-          "id": "sqlite-build-amalgamation",
-          "category": "config-setup",
-          "question": "What is the SQLite amalgamation and how is it produced?",
-          "ground_truth": "The amalgamation is the single-file distribution form of SQLite, sqlite3.c, assembled from the individual sources under src/ during the build. The public C API is declared in the src/sqlite.h.in template, which is expanded into the sqlite3.h header shipped with the amalgamation.",
-          "expected_sources": ["src/sqlite.h.in", "Makefile.in", "AGENTS.md"],
-          "must_include_any": ["amalgamation", "sqlite3.c", "sqlite.h.in"]
-        },
-        {
-          "id": "sqlite-followup-wal",
-          "category": "conversation",
-          "turns": [
-            {"role": "user", "content": "How does SQLite guarantee that transactions survive a crash?"},
-            {"role": "assistant", "content": "Durability is enforced through the pager and, depending on journal mode, the write-ahead log."}
-          ],
-          "question": "which files implement that WAL behavior?",
-          "ground_truth": "WAL-mode durability is implemented mainly in src/wal.c, working together with src/pager.c, which coordinates pager and journal behavior, and the OS-level VFS layer such as src/os_unix.c, which performs the actual fsync/durability calls.",
-          "expected_sources": ["src/wal.c", "src/pager.c", "src/os_unix.c"],
-          "must_include_any": ["wal.c", "pager.c", "fsync", "VFS"]
+          "must_include_any": ["packages/signing", "transports", "seal-document", "PDF", "signing"]
         }
       ]
     },
@@ -159,7 +84,7 @@
           "id": "fastapi-purpose",
           "category": "architecture",
           "question": "What is FastAPI and what is it built on top of?",
-          "ground_truth": "FastAPI is a Python web framework for building APIs. It is built on top of Starlette for the web-facing parts and Pydantic for data validation and serialization, and it automatically generates an OpenAPI schema along with interactive Swagger UI and ReDoc documentation.",
+          "ground_truth": "FastAPI is a Python web framework for building APIs. It builds on Starlette for web functionality and Pydantic for data validation and serialization, and it generates OpenAPI-based documentation.",
           "expected_sources": ["README.md", "fastapi/applications.py", "pyproject.toml"],
           "must_include_any": ["Starlette", "Pydantic", "OpenAPI"]
         },
@@ -167,7 +92,7 @@
           "id": "fastapi-app-class",
           "category": "specific-function",
           "question": "What does the FastAPI application class do and where is it defined?",
-          "ground_truth": "The central FastAPI class is defined in fastapi/applications.py. It ties together routing, dependency injection, middleware, exception handling, and OpenAPI schema generation for the whole application.",
+          "ground_truth": "The central FastAPI class is defined in fastapi/applications.py and coordinates application routing, middleware, exception handling, dependency-related behavior, and OpenAPI generation.",
           "expected_sources": ["fastapi/applications.py"],
           "must_include_any": ["applications.py", "routing", "OpenAPI"]
         },
@@ -175,7 +100,7 @@
           "id": "fastapi-routing",
           "category": "implementation",
           "question": "Where is path operation routing implemented in FastAPI?",
-          "ground_truth": "Routing is implemented in fastapi/routing.py, which defines APIRoute and APIRouter. It handles path matching, resolves dependencies for each incoming request, and serializes the response for every registered endpoint.",
+          "ground_truth": "Path operation routing is implemented in fastapi/routing.py, including APIRoute and APIRouter and request handling that integrates dependency resolution and response serialization.",
           "expected_sources": ["fastapi/routing.py"],
           "must_include_any": ["APIRoute", "APIRouter", "routing.py"]
         },
@@ -183,15 +108,15 @@
           "id": "fastapi-dependency-injection",
           "category": "cross-file",
           "question": "How does FastAPI resolve dependencies declared with Depends()?",
-          "ground_truth": "Depends() and related parameter markers are defined in fastapi/params.py and fastapi/param_functions.py. The actual dependency tree resolution for each request happens in fastapi/dependencies/utils.py, which fastapi/routing.py calls while handling a request.",
+          "ground_truth": "Depends-related parameter declarations are defined in FastAPI's parameter modules, while dependency tree construction and request-time resolution live under fastapi/dependencies and are integrated into routing.",
           "expected_sources": ["fastapi/dependencies/utils.py", "fastapi/params.py", "fastapi/param_functions.py", "fastapi/routing.py"],
-          "must_include_any": ["Depends", "dependencies/utils.py", "dependency"]
+          "must_include_any": ["Depends", "dependencies/utils.py", "dependency", "routing.py"]
         },
         {
           "id": "fastapi-openapi-generation",
           "category": "api",
           "question": "How does FastAPI generate the OpenAPI schema and interactive docs?",
-          "ground_truth": "The OpenAPI JSON schema is generated in fastapi/openapi/utils.py from the app's routes and Pydantic models. The Swagger UI and ReDoc HTML pages are served through helper functions such as get_swagger_ui_html and get_redoc_html in fastapi/openapi/docs.py.",
+          "ground_truth": "FastAPI generates its OpenAPI schema from routes and models using its OpenAPI utilities, while Swagger UI and ReDoc HTML are produced by helpers in the OpenAPI docs module.",
           "expected_sources": ["fastapi/openapi/utils.py", "fastapi/openapi/docs.py"],
           "must_include_any": ["OpenAPI", "Swagger", "ReDoc", "openapi/utils.py"]
         },
@@ -199,7 +124,7 @@
           "id": "fastapi-error-handling",
           "category": "error-handling",
           "question": "How does FastAPI turn validation failures and raised exceptions into HTTP responses?",
-          "ground_truth": "FastAPI defines HTTPException and RequestValidationError in fastapi/exceptions.py. The default handlers that convert those exceptions into JSON error responses are registered in fastapi/exception_handlers.py.",
+          "ground_truth": "FastAPI defines framework exceptions such as HTTPException and RequestValidationError and provides default exception handlers that convert them into HTTP responses.",
           "expected_sources": ["fastapi/exceptions.py", "fastapi/exception_handlers.py"],
           "must_include_any": ["HTTPException", "RequestValidationError", "exception_handlers.py"]
         },
@@ -207,7 +132,7 @@
           "id": "fastapi-security",
           "category": "specific-function",
           "question": "How does FastAPI support authentication schemes like OAuth2 and HTTP Bearer tokens?",
-          "ground_truth": "Authentication helpers live under fastapi/security, which provides classes such as OAuth2PasswordBearer and HTTPBearer. They integrate with the dependency injection system and are automatically reflected in the generated OpenAPI security schema.",
+          "ground_truth": "FastAPI provides authentication helpers under fastapi/security, including OAuth2 and HTTP bearer classes. They integrate with dependency injection and contribute security information to OpenAPI generation.",
           "expected_sources": ["fastapi/security", "fastapi/openapi/utils.py"],
           "must_include_any": ["OAuth2", "HTTPBearer", "security", "dependency"]
         },
@@ -216,14 +141,89 @@
           "category": "conversation",
           "turns": [
             {"role": "user", "content": "How does FastAPI encode response data before it goes back to the client?"},
-            {"role": "assistant", "content": "Response bodies are converted with jsonable_encoder before being serialized, factoring in the declared response_model."}
+            {"role": "assistant", "content": "Response data can be converted with jsonable_encoder before serialization, taking declared response behavior into account."}
           ],
-          "question": "where is that encoder implemented and how is it tested?",
-          "ground_truth": "jsonable_encoder is implemented in fastapi/encoders.py, converting Pydantic models and other Python objects into JSON-compatible structures. Its behavior is covered by the pytest suite under the tests directory.",
+          "question": "where is jsonable_encoder implemented and where is that behavior tested?",
+          "ground_truth": "jsonable_encoder is implemented in fastapi/encoders.py and converts supported Python and Pydantic values into JSON-compatible structures. Its behavior is exercised by the test suite.",
           "expected_sources": ["fastapi/encoders.py", "tests"],
           "must_include_any": ["jsonable_encoder", "encoders.py", "tests"]
         }
       ]
+    },
+    {
+      "id": "django",
+      "name": "Django",
+      "github_url": "https://github.com/django/django.git",
+      "cases": [
+        {
+          "id": "django-purpose",
+          "category": "architecture",
+          "question": "What is Django and how is the repository organized at a high level?",
+          "ground_truth": "Django is a high-level Python web framework. Framework implementation code lives under django/, documentation under docs/, and the main test suite under tests/.",
+          "expected_sources": ["README.rst", "django", "docs", "tests"],
+          "must_include_any": ["web framework", "django", "docs", "tests"]
+        },
+        {
+          "id": "django-url-routing",
+          "category": "specific-function",
+          "question": "Where is Django's URL routing machinery implemented and what does it do?",
+          "ground_truth": "Django's URL routing machinery is implemented under django/urls. It provides URL configuration and resolution that maps incoming request paths to views.",
+          "expected_sources": ["django/urls", "django/urls/resolvers.py"],
+          "must_include_any": ["django/urls", "resolver", "URL", "view"]
+        },
+        {
+          "id": "django-request-response",
+          "category": "cross-file",
+          "question": "How does an incoming HTTP request reach a Django view and become a response?",
+          "ground_truth": "Django request handling crosses the core handler layer, middleware processing, URL resolution, the selected view, and HTTP response classes before the response is returned.",
+          "expected_sources": ["django/core/handlers", "django/urls", "django/http"],
+          "must_include_any": ["handler", "middleware", "URL", "view", "response"]
+        },
+        {
+          "id": "django-queryset",
+          "category": "specific-function",
+          "question": "Where is Django's QuerySet implemented and what role does it play?",
+          "ground_truth": "QuerySet is implemented in django/db/models/query.py. It is a central ORM abstraction representing database queries and collections of model instances and supports operations such as filtering and ordering.",
+          "expected_sources": ["django/db/models/query.py"],
+          "must_include_any": ["QuerySet", "query.py", "ORM", "database"]
+        },
+        {
+          "id": "django-settings",
+          "category": "config-setup",
+          "question": "How does Django load and expose project settings?",
+          "ground_truth": "Django's settings machinery lives under django/conf. The settings proxy loads configuration from the configured project settings module and exposes it to framework and application code.",
+          "expected_sources": ["django/conf", "django/conf/__init__.py"],
+          "must_include_any": ["settings", "django/conf", "SETTINGS_MODULE", "configuration"]
+        },
+        {
+          "id": "django-tests",
+          "category": "tests",
+          "question": "Where is Django's main test suite and where are contributors told how to run it?",
+          "ground_truth": "Django keeps its main test suite under tests/. The repository README points contributors to the unit-test instructions under docs/internals/contributing/writing-code/unit-tests.txt.",
+          "expected_sources": ["tests", "README.rst", "docs/internals/contributing/writing-code/unit-tests.txt"],
+          "must_include_any": ["tests", "unit-tests.txt", "test suite"]
+        },
+        {
+          "id": "django-security-csrf",
+          "category": "error-handling",
+          "question": "Where is Django's CSRF protection implemented and what does it validate?",
+          "ground_truth": "Django's CSRF protection is implemented in django/middleware/csrf.py. The middleware validates CSRF-related request state and tokens and rejects requests that fail its checks.",
+          "expected_sources": ["django/middleware/csrf.py"],
+          "must_include_any": ["CSRF", "middleware", "token", "csrf.py"]
+        },
+        {
+          "id": "django-followup-template",
+          "category": "conversation",
+          "turns": [
+            {"role": "user", "content": "How does Django render HTML templates?"},
+            {"role": "assistant", "content": "Django's built-in template system loads templates through configured engines and renders them with a context."}
+          ],
+          "question": "where is that built-in template engine implemented?",
+          "ground_truth": "Django's built-in template engine is implemented under django/template, including engine configuration, parsing, loading, and rendering machinery.",
+          "expected_sources": ["django/template", "django/template/engine.py", "django/template/base.py"],
+          "must_include_any": ["django/template", "Engine", "Template", "render"]
+        }
+      ]
     }
   ]
 }

src/code_parser.py

@@ -14,8 +14,16 @@ LANGUAGE_BY_EXTENSION = {
     ".java": "java",
     ".go": "go",
     ".rs": "rust",
+    ".c": "c",
+    ".h": "c",
+    ".cc": "cpp",
+    ".cpp": "cpp",
+    ".cxx": "cpp",
+    ".hpp": "cpp",
+    ".hh": "cpp",
 }
 
+# Node types that make a good standalone retrieval unit (function/method-sized).
 SYMBOL_NODE_TYPES = {
     "python": {"function_definition", "class_definition"},
     "javascript": {
@@ -62,6 +70,39 @@ SYMBOL_NODE_TYPES = {
         "enum_item",
         "trait_item",
     },
+    "c": {
+        "function_definition",
+        "struct_specifier",
+        "enum_specifier",
+        "type_definition",
+    },
+    "cpp": {
+        "function_definition",
+        "class_specifier",
+        "struct_specifier",
+        "enum_specifier",
+        "namespace_definition",
+        "type_definition",
+    },
+}
+
+# Node types that are "containers": they hold nested members (methods, fields)
+# that are themselves worth indexing as their own chunks. For these, we emit a
+# compact *overview* chunk (signature + docstring + head-of-body + member
+# list) instead of dumping the entire body into one chunk. Without this, a
+# large class becomes a single multi-hundred-line chunk whose embedding is a
+# blurry average of everything inside it (hurts hit rate for method-specific
+# queries) and whose content is silently cut off by the ~1500 char preview
+# used when the chunk is fed to the LLM (hurts faithfulness, since the method
+# the user actually asked about may fall outside the truncated window).
+CONTAINER_NODE_TYPES = {
+    "python": {"class_definition"},
+    "javascript": {"class_declaration"},
+    "typescript": {"class_declaration"},
+    "tsx": {"class_declaration"},
+    "java": {"class_declaration", "interface_declaration", "enum_declaration"},
+    "rust": {"impl_item", "trait_item"},
+    "cpp": {"class_specifier", "namespace_definition"},
 }
 
 IDENTIFIER_TYPES = {
@@ -71,6 +112,10 @@ IDENTIFIER_TYPES = {
     "field_identifier",
 }
 
+MAX_OVERVIEW_BODY_LINES = 40
+MAX_OVERVIEW_CHARS = 1200
+MAX_MEMBERS_LISTED = 25
+
 
 class CodeParser:
     def __init__(self):
@@ -102,58 +147,282 @@ class CodeParser:
         lines = source.splitlines()
         chunks = []
         capture_types = SYMBOL_NODE_TYPES.get(language, set())
+        container_types = CONTAINER_NODE_TYPES.get(language, set())
 
-        def visit(node):
+        def visit(node, context_name: Optional[str] = None):
             if node.type in capture_types:
-                chunk = self._build_chunk(node, source, lines, relative_path, language)
+                if node.type in container_types:
+                    overview = self._build_container_overview(
+                        node, source, lines, relative_path, language, context_name, capture_types
+                    )
+                    if overview:
+                        chunks.append(overview)
+
+                    own_name = self._extract_own_name(node, source)
+                    nested_context = own_name
+                    if context_name and own_name:
+                        nested_context = f"{context_name}.{own_name}"
+                    elif context_name and not own_name:
+                        nested_context = context_name
+
+                    for child in node.children:
+                        visit(child, nested_context)
+                    return
+
+                chunk = self._build_chunk(node, source, lines, relative_path, language, context_name)
                 if chunk:
                     chunks.append(chunk)
                     return
             for child in node.children:
-                visit(child)
+                visit(child, context_name)
 
         visit(tree.root_node)
 
         if not chunks:
-            return self._fallback_chunks(source, relative_path, language)
+            chunks = self._fallback_chunks(source, relative_path, language)
+
+        file_overview = self._build_file_overview(
+            tree.root_node, source, lines, relative_path, language, chunks
+        )
+        if file_overview:
+            chunks.insert(0, file_overview)
 
         return chunks
 
-    def _build_chunk(self, node, source: str, lines: List[str], relative_path: str, language: str) -> Optional[Dict]:
+    def _build_chunk(
+        self,
+        node,
+        source: str,
+        lines: List[str],
+        relative_path: str,
+        language: str,
+        context_name: Optional[str] = None,
+    ) -> Optional[Dict]:
         start_line = node.start_point[0] + 1
         end_line = node.end_point[0] + 1
         snippet = "\n".join(lines[start_line - 1 : end_line]).strip()
         if len(snippet.splitlines()) < 2:
             return None
 
-        name_node = node.child_by_field_name("name")
-        symbol_name = None
-        if name_node is not None:
-            symbol_name = source[name_node.start_byte : name_node.end_byte].strip()
-        if not symbol_name:
-            symbol_name = self._find_identifier(node, source)
+        own_name = self._extract_own_name(node, source)
+        qualified_name = own_name
+        if context_name and own_name:
+            qualified_name = f"{context_name}.{own_name}"
+        elif context_name and not own_name:
+            qualified_name = context_name
 
         signature = lines[start_line - 1].strip() if start_line - 1 < len(lines) else ""
         searchable_text = "\n".join(
-            part for part in [relative_path, symbol_name or "", signature, snippet] if part
+            part
+            for part in [relative_path, context_name or "", qualified_name or "", signature, snippet]
+            if part
         )
 
+        metadata = {"parser": "tree-sitter"}
+        if context_name:
+            metadata["parent"] = context_name
+
         return {
             "file_path": relative_path,
             "language": language,
-            "symbol_name": symbol_name or relative_path.split("/")[-1],
+            "symbol_name": qualified_name or relative_path.split("/")[-1],
             "symbol_type": node.type,
             "line_start": start_line,
             "line_end": end_line,
             "signature": signature,
             "content": snippet,
             "searchable_text": searchable_text,
+            "metadata_json": metadata,
+        }
+
+    def _build_container_overview(
+        self,
+        node,
+        source: str,
+        lines: List[str],
+        relative_path: str,
+        language: str,
+        context_name: Optional[str],
+        capture_types: set,
+    ) -> Optional[Dict]:
+        start_line = node.start_point[0] + 1
+        end_line = node.end_point[0] + 1
+        body_lines = lines[start_line - 1 : end_line]
+        if not body_lines:
+            return None
+
+        own_name = self._extract_own_name(node, source)
+        qualified_name = own_name
+        if context_name and own_name:
+            qualified_name = f"{context_name}.{own_name}"
+        elif context_name and not own_name:
+            qualified_name = context_name
+        if not qualified_name:
+            qualified_name = relative_path.split("/")[-1]
+
+        # Head-of-body preview: naturally captures the signature, docstring,
+        # and field declarations that come before the first nested method,
+        # so simple data classes / structs keep their field list even though
+        # we no longer store the entire body verbatim.
+        preview_lines = body_lines[:MAX_OVERVIEW_BODY_LINES]
+        content = "\n".join(preview_lines).strip()
+        truncated_body = len(body_lines) > MAX_OVERVIEW_BODY_LINES
+        if len(content) > MAX_OVERVIEW_CHARS:
+            content = content[:MAX_OVERVIEW_CHARS].rstrip()
+            truncated_body = True
+
+        member_names = self._collect_member_names(node, source, capture_types)
+        members_line = ""
+        if member_names:
+            shown = member_names[:MAX_MEMBERS_LISTED]
+            members_line = f"Members: {', '.join(shown)}"
+            remaining = len(member_names) - len(shown)
+            if remaining > 0:
+                members_line += f" (+{remaining} more, indexed separately)"
+        elif truncated_body:
+            members_line = "(body truncated; see file for full contents)"
+
+        content_parts = [part for part in [content, members_line] if part]
+        full_content = "\n\n".join(content_parts)
+
+        signature = body_lines[0].strip() if body_lines else ""
+        searchable_text = "\n".join(
+            part
+            for part in [relative_path, context_name or "", qualified_name, signature, full_content]
+            if part
+        )
+
+        return {
+            "file_path": relative_path,
+            "language": language,
+            "symbol_name": qualified_name,
+            "symbol_type": f"{node.type}_overview",
+            "line_start": start_line,
+            "line_end": end_line,
+            "signature": signature,
+            "content": full_content,
+            "searchable_text": searchable_text,
             "metadata_json": {
                 "parser": "tree-sitter",
+                "kind": "container_overview",
+                **({"parent": context_name} if context_name else {}),
             },
         }
 
-    def _find_identifier(self, node, source: str) -> Optional[str]:
+    def _build_file_overview(self, root_node, source: str, lines: List[str], relative_path: str, language: str, symbol_chunks: List[Dict]) -> Optional[Dict]:
+        if language == "text":
+            return None
+        path = Path(relative_path)
+        parts = list(path.parts)
+        directory = str(path.parent) if str(path.parent) != "." else "repository root"
+        filename = path.name
+        component = parts[1] if len(parts) >= 2 and parts[0] in {"packages", "apps"} else (parts[-2] if len(parts) >= 2 else "")
+        role = self._infer_file_role(relative_path)
+        imports, exports, export_targets = self._extract_module_edges(root_node, source)
+        symbols, seen = [], set()
+        for chunk in symbol_chunks:
+            name = chunk.get("symbol_name")
+            if name and name not in seen and chunk.get("symbol_type") != "fallback_chunk":
+                seen.add(name); symbols.append(name)
+            if len(symbols) >= 40:
+                break
+        overview = [f"File: {relative_path}", f"Filename: {filename}", f"Directory: {directory}", f"Language: {language}"]
+        if component: overview.append(f"Package/component: {component}")
+        overview.append(f"Role: {role}")
+        if imports: overview.append("Imports: " + ", ".join(imports[:30]))
+        if exports: overview.append("Exports: " + ", ".join(exports[:40]))
+        if export_targets: overview.append("Export targets: " + ", ".join(export_targets[:30]))
+        if symbols: overview.append("Symbols: " + ", ".join(symbols))
+        content = "\n".join(overview)
+        path_terms = " ".join(t for t in re.split(r"[\/._:-]+", relative_path) if t)
+        return {
+            "file_path": relative_path, "language": language,
+            "symbol_name": f"{filename}:module-overview", "symbol_type": "file_overview",
+            "line_start": 1, "line_end": 1, "signature": f"module {relative_path}",
+            "content": content,
+            "searchable_text": content + f"\nPath terms: {path_terms}\nModule: {path.stem}",
+            "metadata_json": {"parser": "tree-sitter", "kind": "file_overview", "directory": directory, "component": component, "role": role, "imports": imports[:30], "exports": exports[:40], "export_targets": export_targets[:30]},
+        }
+
+    @staticmethod
+    def _infer_file_role(relative_path: str) -> str:
+        path = relative_path.lower().replace("\\", "/")
+        name = Path(path).name
+        if name in {"index.ts", "index.tsx", "index.js", "index.jsx"}:
+            return "module/package entry point or barrel export"
+        for needle, role in [("router", "router/API composition"), ("route", "request route/endpoint"), ("handler", "handler/job execution"), ("controller", "request controller"), ("service", "service/business logic"), ("transport", "transport/integration adapter"), ("adapter", "integration adapter"), ("repository", "data repository"), ("config", "configuration"), ("schema", "schema/type definition"), ("test", "test"), ("spec", "test/specification")]:
+            if needle in name or f"/{needle}" in path:
+                return role
+        return "source module"
+
+    def _extract_module_edges(self, root_node, source: str):
+        imports, exports, targets = [], [], []
+        for child in root_node.children:
+            text = source[child.start_byte:child.end_byte].strip()
+            if not text: continue
+            if child.type == "import_statement" or text.startswith("import "):
+                target = self._module_target(text)
+                if target and target not in imports: imports.append(target)
+            if "export" in child.type or text.startswith("export ") or text.startswith("module.exports"):
+                target = self._module_target(text)
+                if target and target not in targets: targets.append(target)
+                for name in self._exported_names(text):
+                    if name not in exports: exports.append(name)
+        return imports, exports, targets
+
+    @staticmethod
+    def _module_target(statement: str) -> Optional[str]:
+        for pattern in [r"from\s+[\"']([^\"']+)[\"']", r"(?:import|export)\s+[\"']([^\"']+)[\"']", r"require\(\s*[\"']([^\"']+)[\"']\s*\)"]:
+            match = re.search(pattern, statement)
+            if match: return match.group(1)
+        return None
+
+    @staticmethod
+    def _exported_names(statement: str) -> List[str]:
+        names = []
+        match = re.search(r"export\s*\{([^}]+)\}", statement, re.S)
+        if match:
+            for part in match.group(1).split(","):
+                item = part.strip()
+                if item: names.append(re.split(r"\s+as\s+", item)[-1].strip())
+        match = re.search(r"export\s+(?:default\s+)?(?:async\s+)?(?:const|let|var|function|class|interface|type|enum)\s+([A-Za-z_$][A-Za-z0-9_$]*)", statement)
+        if match: names.append(match.group(1))
+        if statement.startswith("export default") and not match: names.append("default")
+        if statement.startswith("export *"): names.append("*")
+        return names
+
+    def _collect_member_names(self, node, source: str, capture_types: set) -> List[str]:
+        """Collect names of direct member symbols (methods/fields) inside a
+        container without descending into nested containers, so a class's
+        member list doesn't pick up grandchildren from an inner class."""
+        names = []
+        seen = set()
+
+        def walk(current):
+            for child in current.children:
+                if child.type in capture_types:
+                    name = self._extract_own_name(child, source)
+                    if name and name not in seen:
+                        seen.add(name)
+                        names.append(name)
+                    # Don't descend further into this member's own body.
+                    continue
+                walk(child)
+
+        walk(node)
+        return names
+
+    @staticmethod
+    def _extract_own_name(node, source: str) -> Optional[str]:
+        name_node = node.child_by_field_name("name")
+        if name_node is not None:
+            candidate = source[name_node.start_byte : name_node.end_byte].strip()
+            if candidate:
+                return candidate
+        return CodeParser._find_identifier(node, source)
+
+    @staticmethod
+    def _find_identifier(node, source: str) -> Optional[str]:
         stack = list(node.children)
         while stack:
             current = stack.pop(0)
@@ -220,4 +489,4 @@ class CodeParser:
                         },
                     }
                 )
-        return blocks
+        return blocks

src/embeddings.py

@@ -320,13 +320,17 @@ class EmbeddingGenerator:
         return np.array(embeddings, dtype="float32")
 
     def _build_bedrock_embedding_request(self, texts: List[str], input_type: str) -> dict:
+        max_chars = int(os.getenv("BEDROCK_EMBEDDING_MAX_CHARS", "2000"))
+
         payload = {
-            "texts": texts,
+            "texts": [text[:max_chars] for text in texts],
             "input_type": input_type,
             "embedding_types": ["float"],
         }
+
         if self.bedrock_output_dimensionality:
             payload["output_dimension"] = self.bedrock_output_dimensionality
+
         return payload
 
     def _encode_with_backoff(

src/hybrid_search.py

@@ -1,6 +1,7 @@
 import re
+import threading
 from collections import defaultdict
-from typing import List
+from typing import Dict, List, Optional
 
 from rank_bm25 import BM25Okapi
 from sentence_transformers import CrossEncoder
@@ -9,34 +10,95 @@ TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_./:-]*")
 
 
 def tokenize(text: str) -> List[str]:
-    return [token.lower() for token in TOKEN_RE.findall(text)]
+    raw_tokens = TOKEN_RE.findall(text or "")
+    tokens = []
+
+    for raw in raw_tokens:
+        lowered = raw.lower()
+        tokens.append(lowered)
+
+        # Keep the original code/path token, but also expose its components to
+        # BM25. This makes sendDocument, seal-document.handler.ts, etc. match
+        # natural-language queries much more reliably.
+        pieces = re.split(r"[./:_-]+", raw)
+        for piece in pieces:
+            if not piece:
+                continue
+            tokens.append(piece.lower())
+            camel_parts = re.findall(
+                r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z]|$)|\d+",
+                piece,
+            )
+            tokens.extend(part.lower() for part in camel_parts if part)
+
+    return [token for token in tokens if token]
 
 
 class HybridSearchEngine:
     def __init__(self, reranker_model: str = "cross-encoder/ms-marco-MiniLM-L-6-v2"):
         self.reranker = CrossEncoder(reranker_model)
+        # Per-repo cached BM25 index so a question doesn't have to
+        # re-tokenize and re-build the lexical index over every chunk in the
+        # repo on every single request. Built once when indexing finishes,
+        # evicted when the repo is reset/deleted/expired.
+        self._repo_indexes: Dict[int, dict] = {}
+        self._index_lock = threading.Lock()
 
     def build_for_repository(self, repo_id: int, chunks: List[dict]):
-        return None
+        if not chunks:
+            with self._index_lock:
+                self._repo_indexes.pop(repo_id, None)
+            return
+
+        corpus_tokens = [tokenize(chunk["searchable_text"]) for chunk in chunks]
+        bm25 = BM25Okapi(corpus_tokens) if corpus_tokens else None
+        with self._index_lock:
+            self._repo_indexes[repo_id] = {"bm25": bm25, "chunks": chunks}
 
     def remove_repository(self, repo_id: int):
-        return None
+        with self._index_lock:
+            self._repo_indexes.pop(repo_id, None)
 
-    def bm25_search(self, chunks: List[dict], query: str, top_k: int = 12) -> List[dict]:
+    def bm25_search(
+        self,
+        chunks: List[dict],
+        query: str,
+        top_k: int = 12,
+        repo_id: Optional[int] = None,
+    ) -> List[dict]:
         if not chunks:
             return []
         tokens = tokenize(query)
         if not tokens:
             return []
 
-        corpus_tokens = [tokenize(chunk["searchable_text"]) for chunk in chunks]
-        bm25 = BM25Okapi(corpus_tokens) if corpus_tokens else None
+        bm25 = None
+        source_chunks = chunks
+
+        if repo_id is not None:
+            with self._index_lock:
+                cached = self._repo_indexes.get(repo_id)
+            # Guard against a stale cache (e.g. repo was re-indexed but the
+            # cache write raced with this read) by checking the corpus size
+            # still lines up before trusting it.
+            if cached is not None and len(cached["chunks"]) == len(chunks):
+                bm25 = cached["bm25"]
+                source_chunks = cached["chunks"]
+
+        if bm25 is None:
+            # Fall back to building an ephemeral index. Keeps this method
+            # correct on its own even if build_for_repository wasn't called
+            # first (e.g. direct/test usage), just without the caching win.
+            corpus_tokens = [tokenize(chunk["searchable_text"]) for chunk in chunks]
+            bm25 = BM25Okapi(corpus_tokens) if corpus_tokens else None
+            source_chunks = chunks
+
         if not bm25:
             return []
 
         scores = bm25.get_scores(tokens)
         ranked = sorted(
-            zip(chunks, scores),
+            zip(source_chunks, scores),
             key=lambda item: item[1],
             reverse=True,
         )[:top_k]
@@ -69,11 +131,17 @@ class HybridSearchEngine:
         merged = sorted(fused.values(), key=lambda item: item["rrf_score"], reverse=True)
         return merged[:top_k]
 
-    def rerank(self, query: str, candidates: List[dict], top_k: int = 6) -> List[dict]:
-        """
-        FIX: top_k now defaults to 6 and callers should pass a small final number (4-6),
-        NOT search_depth (which was up to 120). Reranking 120 items then dumping them
-        all into the LLM context was the main faithfulness killer.
+    def rerank(
+        self,
+        query: str,
+        candidates: List[dict],
+        top_k: Optional[int] = None,
+    ) -> List[dict]:
+        """Score candidates with the cross-encoder and optionally truncate.
+
+        Reranking depth is intentionally separate from answer-context depth.
+        Callers can rerank a broad candidate set and still send only a small
+        final source set to the LLM.
         """
         if not candidates:
             return []
@@ -91,7 +159,7 @@ class HybridSearchEngine:
             reranked.append(enriched)
 
         reranked.sort(key=lambda item: item["rerank_score"], reverse=True)
-        return reranked[:top_k]
+        return reranked[:top_k] if top_k is not None else reranked
 
     @staticmethod
     def normalize_semantic_results(results: List[dict]) -> List[dict]:

src/rag_system.py

@@ -278,6 +278,9 @@ class CodebaseRAGSystem:
                 repo.session_expires_at = self._session_expiry()
                 self._mark_repo_updated(repo)
                 self.repo_chunks[repo.id] = serialized
+            # Build the lexical (BM25) index once now, instead of
+            # re-tokenizing every chunk in the repo on every question.
+            self.hybrid_search.build_for_repository(repo.id, serialized)
             self.vector_store.save()
             with self.repo_lock:
                 self.indexing_progress.pop(repo.id, None)
@@ -343,6 +346,7 @@ class CodebaseRAGSystem:
         question: str,
         top_k: int = 8,
         history=None,
+        debug_retrieval: bool = False,
     ) -> dict:
         with self.repo_lock:
             self._cleanup_expired_sessions()
@@ -390,9 +394,17 @@ class CodebaseRAGSystem:
             repo_chunks,
             retrieval_query,
             top_k=search_depth,
+            repo_id=repo_id,
         )
+
         semantic_hits = self.hybrid_search.normalize_semantic_results(semantic_hits)
-        fused = self.hybrid_search.reciprocal_rank_fusion(lexical_hits, semantic_hits, top_k=search_depth)
+        semantic_ranks = self._rank_map(semantic_hits)
+        lexical_ranks = self._rank_map(lexical_hits)
+
+        fused = self.hybrid_search.reciprocal_rank_fusion(
+            lexical_hits, semantic_hits, top_k=search_depth
+        )
+        fused_ranks = self._rank_map(fused)
 
         path_hits = self._path_intent_search(
             repo_chunks,
@@ -400,26 +412,67 @@ class CodebaseRAGSystem:
             retrieval_query,
             top_k=search_depth,
         )
-        fused = self._merge_ranked_candidates(fused, path_hits, top_k=search_depth)
+        path_ranks = self._rank_map(path_hits)
+
+        merged = self._merge_ranked_candidates(fused, path_hits, top_k=search_depth)
 
         rerank_query = retrieval_query if question_intent in deep_search_intents else question
 
-        # FIX: rerank to a small candidate pool first (20), then let
-        # _prioritize_results and _select_answer_sources trim to final top_k.
-        # Previously rerank was called with search_depth (up to 120), meaning
-        # the LLM received far too many chunks and faithfulness dropped.
-        rerank_pool = min(search_depth, 20)
-        reranked = self.hybrid_search.rerank(rerank_query, fused, top_k=rerank_pool)
+        # Rerank broadly for recall. This is independent of the final LLM
+        # context size, which remains capped below.
+        rerank_pool = min(search_depth, 50)
+        reranked = self.hybrid_search.rerank(rerank_query, merged, top_k=rerank_pool)
+        rerank_ranks = self._rank_map(reranked)
 
-        reranked = self._prioritize_results(question, retrieval_query, reranked, top_k=top_k)
+        prioritized = self._prioritize_results(
+            question, retrieval_query, reranked, top_k=top_k
+        )
 
-        # FIX: cap final sources at 5 instead of top_k (8).
-        # 5 sources × 1500 chars = ~7500 chars context, which the LLM handles well.
-        # 8 sources × 2500 chars = ~20000 chars, which causes lost-in-the-middle issues.
         final_top_k = min(top_k, 5)
-        reranked = self._select_answer_sources(question, reranked, top_k=final_top_k)
+        final_sources = self._select_answer_sources(
+            question, prioritized, top_k=final_top_k
+        )
+        final_ranks = self._rank_map(final_sources)
+
+        retrieval_debug = []
+        if debug_retrieval:
+            all_candidates = {}
+            for stage_items in (semantic_hits, lexical_hits, fused, path_hits, merged, reranked, prioritized, final_sources):
+                for item in stage_items:
+                    all_candidates[item["id"]] = {**all_candidates.get(item["id"], {}), **item}
+
+            for chunk_id, item in all_candidates.items():
+                retrieval_debug.append(
+                    {
+                        "id": chunk_id,
+                        "file_path": item.get("file_path"),
+                        "symbol_name": item.get("symbol_name"),
+                        "semantic_rank": semantic_ranks.get(chunk_id),
+                        "bm25_rank": lexical_ranks.get(chunk_id),
+                        "fused_rank": fused_ranks.get(chunk_id),
+                        "path_rank": path_ranks.get(chunk_id),
+                        "rerank_rank": rerank_ranks.get(chunk_id),
+                        "final_rank": final_ranks.get(chunk_id),
+                        "semantic_score": item.get("semantic_score"),
+                        "bm25_score": item.get("bm25_score"),
+                        "rrf_score": item.get("rrf_score"),
+                        "path_score": item.get("path_score"),
+                        "rerank_score": item.get("rerank_score"),
+                        "final_score": item.get("final_score"),
+                    }
+                )
+
+            retrieval_debug.sort(
+                key=lambda item: (
+                    item["final_rank"] is None,
+                    item["final_rank"] or 10**9,
+                    item["rerank_rank"] or 10**9,
+                )
+            )
 
-        answer = self._generate_answer(repo, question, reranked, normalized_history)
+        answer = self._generate_answer(repo, question, final_sources, normalized_history)
+        if debug_retrieval:
+            answer["retrieval_debug"] = retrieval_debug
         return answer
 
     def end_session(self, session_key: str):
@@ -928,6 +981,29 @@ Do not leave the answer unfinished.
             parts.append(f"Previous answer: {recent_assistant[0][:300]}")
         return "\n".join(parts)
 
+    @staticmethod
+    def _rank_map(items: List[dict]) -> Dict[str, int]:
+        return {item["id"]: rank for rank, item in enumerate(items, start=1)}
+
+    @staticmethod
+    def _minmax(values: List[float]) -> List[float]:
+        if not values:
+            return []
+        low = min(values)
+        high = max(values)
+        if high == low:
+            return [0.0 for _ in values]
+        return [(value - low) / (high - low) for value in values]
+
+    @staticmethod
+    def _source_family(file_path: str) -> str:
+        parts = (file_path or "").strip("/").split("/")
+        if not parts or not parts[0]:
+            return ""
+        if parts[0] in {"packages", "apps"} and len(parts) >= 2:
+            return "/".join(parts[:2])
+        return parts[0]
+
     def _merge_ranked_candidates(
         self,
         ranked_results: List[dict],
@@ -1036,6 +1112,9 @@ Do not leave the answer unfinished.
         results: List[dict],
         top_k: int,
     ) -> List[dict]:
+        if not results:
+            return []
+
         combined_query = f"{question} {retrieval_query}".lower()
         wants_code = any(
             token in combined_query
@@ -1046,41 +1125,47 @@ Do not leave the answer unfinished.
             "docs",
             "overview",
         }
-        wants_repo_overview = self._is_repo_overview_question(question) or self._is_repo_overview_question(
-            retrieval_query
+        wants_repo_overview = self._is_repo_overview_question(
+            question
+        ) or self._is_repo_overview_question(retrieval_query)
+
+        rerank_norm = self._minmax([float(item.get("rerank_score", 0.0)) for item in results])
+        rrf_norm = self._minmax([float(item.get("rrf_score", 0.0)) for item in results])
+        path_norm = self._minmax([float(item.get("path_score", 0.0)) for item in results])
+        canonical_norm = self._minmax(
+            [float(self._canonical_path_priority(item, combined_query)) for item in results]
         )
 
-        def sort_key(item: dict):
+        scored = []
+        for index, item in enumerate(results):
+            enriched = dict(item)
             is_doc = self._is_doc_source(item)
-            return (
-                self._canonical_path_priority(item, combined_query),
-                float(item.get("path_score", 0.0)),
-                self._doc_priority(item),
-                1 if wants_repo_overview and is_doc else 0,
-                1 if (wants_docs and is_doc) or (not wants_docs and not is_doc) else 0,
-                1 if wants_code and not is_doc else 0,
-                1 if question_intent in {"api", "implementation", "cross_file", "error_handling", "setup"} and not is_doc else 0,
-                float(item.get("rerank_score", 0.0)),
-                float(item.get("semantic_score", 0.0)),
-                float(item.get("bm25_score", 0.0)),
+            intent_bonus = 0.0
+
+            if wants_repo_overview and is_doc:
+                intent_bonus += 1.0
+            if wants_docs and is_doc:
+                intent_bonus += 0.8
+            if wants_code and not is_doc:
+                intent_bonus += 0.5
+            if (
+                question_intent
+                in {"api", "implementation", "cross_file", "error_handling", "setup"}
+                and not is_doc
+            ):
+                intent_bonus += 0.4
+
+            enriched["final_score"] = (
+                0.50 * rerank_norm[index]
+                + 0.20 * rrf_norm[index]
+                + 0.12 * path_norm[index]
+                + 0.10 * canonical_norm[index]
+                + 0.08 * min(intent_bonus, 1.0)
             )
+            scored.append(enriched)
 
-        ranked = sorted(results, key=sort_key, reverse=True)
-        if wants_docs or wants_repo_overview:
-            return ranked[:top_k]
-
-        selected = []
-        doc_items = []
-        for item in ranked:
-            if self._is_doc_source(item):
-                doc_items.append(item)
-                continue
-            selected.append(item)
-            if len(selected) == top_k:
-                return selected
-
-        selected.extend(doc_items[: max(1, top_k - len(selected))])
-        return selected[:top_k]
+        scored.sort(key=lambda item: item["final_score"], reverse=True)
+        return scored[:top_k]
 
     def _select_answer_sources(
         self,
@@ -1094,14 +1179,33 @@ Do not leave the answer unfinished.
         intent = self._question_intent(question)
         max_per_file = 2 if intent in {"overview", "docs"} else 1
         selected = []
+        selected_ids = set()
         file_counts = {}
+        used_families = set()
+
+        if intent == "cross_file":
+            for item in results:
+                file_path = item.get("file_path", "")
+                family = self._source_family(file_path)
+                if family and family in used_families:
+                    continue
+                selected.append(item)
+                selected_ids.add(item["id"])
+                if family:
+                    used_families.add(family)
+                file_counts[file_path] = 1
+                if len(selected) == top_k:
+                    return selected
 
         for item in results:
+            if item["id"] in selected_ids:
+                continue
             file_path = item.get("file_path", "")
             count = file_counts.get(file_path, 0)
             if count >= max_per_file:
                 continue
             selected.append(item)
+            selected_ids.add(item["id"])
             file_counts[file_path] = count + 1
             if len(selected) == top_k:
                 break
@@ -1260,27 +1364,44 @@ Do not leave the answer unfinished.
     @staticmethod
     def _is_repo_overview_question(question: str) -> bool:
         normalized = " ".join((question or "").lower().split())
-        return any(
-            phrase in normalized
-            for phrase in {
-                "what is the repo about",
-                "what is this repo about",
-                "what does the repo do",
-                "what does this repo do",
-                "what is the repository about",
-                "what does the repository do",
-                "what is this project about",
-                "what does this project do",
-                "repo summary",
-                "repository summary",
-                "project summary",
-                "summarize the repo",
-                "summarize this repo",
-                "repo overview",
-                "repository overview",
-                "project overview",
-            }
+        explicit_phrases = {
+            "what is the repo about",
+            "what is this repo about",
+            "what does the repo do",
+            "what does this repo do",
+            "what is the repository about",
+            "what does the repository do",
+            "what is this project about",
+            "what does this project do",
+            "repo summary",
+            "repository summary",
+            "project summary",
+            "summarize the repo",
+            "summarize this repo",
+            "repo overview",
+            "repository overview",
+            "project overview",
+        }
+        if any(phrase in normalized for phrase in explicit_phrases):
+            return True
+
+        code_markers = {
+            "function", "class", "method", "endpoint", "api", "router",
+            "route", "implementation", "implemented", "file", "package",
+            "module", "where", "tests",
+        }
+        if any(re.search(rf"(?<![a-z0-9]){re.escape(marker)}(?![a-z0-9])", normalized) for marker in code_markers):
+            return False
+
+        purpose_patterns = (
+            r"^what is [\w.-]+(?:\s+and\s+what\s+.+)?\??$",
+            r"^what does [\w.-]+ do\??$",
+            r"^what problem does [\w.-]+ solve\??$",
+            r"^what product problem .+ solve\??$",
+            r"^(?:describe|explain) [\w.-]+\??$",
+            r"^(?:what is|explain|describe) the (?:purpose|project|repository)\b",
         )
+        return any(re.search(pattern, normalized) for pattern in purpose_patterns)
 
     @staticmethod
     def _is_doc_source(item: dict) -> bool:
@@ -1301,189 +1422,104 @@ Do not leave the answer unfinished.
 
     @staticmethod
     def _domain_path_hints(query: str) -> List[str]:
+        """Map generic question concepts to the directory/file naming
+        conventions real-world repos tend to use for that concept.
+
+        This intentionally stays convention-level (not tied to any single
+        project's folder layout) because this system indexes arbitrary
+        GitHub repositories: a hint list hardcoded to one repo's paths would
+        never match anything in any other repo, silently doing nothing for
+        almost every user while looking like it's helping. These patterns
+        are matched as path *substrings* by the callers, so they work across
+        Python/JS/TS/Go/Java/Rust project layouts without needing to know
+        the specific repo's structure in advance.
+        """
         normalized = " ".join((query or "").lower().split())
         hints = []
 
         def has_any(terms: set[str]) -> bool:
-            matched = False
-            for term in terms:
-                if term.startswith("/"):
-                    matched = matched or term in normalized
-                    continue
-                matched = matched or bool(
-                    re.search(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", normalized)
-                )
-            return matched
+            return any(
+                bool(re.search(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", normalized))
+                for term in terms
+            )
 
         rules = [
             (
-                {"email", "emails", "mailer", "notification", "template"},
-                [
-                    "packages/email",
-                    "packages/lib/server-only/email",
-                    "packages/lib/jobs/definitions/emails",
-                ],
+                {"auth", "authentication", "authorization", "login", "session", "bearer", "oauth", "jwt", "token"},
+                ["auth", "authentication", "authorization", "login", "session", "middleware/auth"],
             ),
             (
-                {"signing", "certificate", "gcloud", "hsm", "p12", "pdf signing"},
-                [
-                    "packages/signing",
-                    "packages/signing/transports",
-                    "packages/signing/helpers",
-                    "packages/lib/jobs/definitions/internal/seal-document",
-                ],
+                {"api", "endpoint", "route", "router", "controller", "handler", "rest", "graphql", "trpc"},
+                ["api", "routes", "routers", "controllers", "handlers", "endpoints", "resolvers"],
             ),
             (
-                {"job", "jobs", "background", "inngest", "bullmq", "queue"},
-                [
-                    "packages/lib/jobs",
-                    "packages/lib/jobs/client",
-                    "packages/lib/jobs/definitions",
-                    "apps/remix/server/router.ts",
-                ],
+                {"database", "db", "sql", "orm", "migration", "migrations", "schema", "model", "models"},
+                ["models", "schema", "migrations", "db", "database", "entities", "repositories", "prisma"],
             ),
             (
-                {"webhook", "webhooks", "ssrf"},
-                [
-                    "packages/lib/server-only/webhooks",
-                    "packages/lib/jobs/definitions/internal/execute-webhook",
-                ],
+                {"test", "tests", "testing", "pytest", "spec", "e2e", "unit test", "integration test"},
+                ["test", "tests", "__tests__", "spec", "e2e", "testing"],
             ),
             (
-                {"recipient", "recipients"},
+                {"config", "configuration", "env", "environment", "settings", "setup", "install", "installation"},
                 [
-                    "packages/lib/server-only/recipient",
-                    "packages/trpc/server/recipient-router",
+                    "config",
+                    "settings",
+                    ".env.example",
+                    "readme.md",
+                    "package.json",
+                    "pyproject.toml",
+                    "docker-compose",
+                    "dockerfile",
                 ],
             ),
             (
-                {"field", "fields"},
-                [
-                    "packages/lib/server-only/field",
-                    "packages/trpc/server/field-router",
-                    "packages/lib/universal/field-renderer",
-                ],
+                {"job", "jobs", "background", "worker", "workers", "queue", "task", "tasks", "cron", "scheduler"},
+                ["jobs", "workers", "queue", "tasks", "scheduler"],
             ),
             (
-                {"template", "templates"},
-                [
-                    "packages/lib/server-only/template",
-                    "packages/trpc/server/template-router",
-                ],
+                {"webhook", "webhooks", "callback", "callbacks"},
+                ["webhook", "webhooks", "callbacks"],
             ),
             (
-                {"envelope", "envelopes"},
-                [
-                    "packages/lib/server-only/envelope",
-                    "packages/lib/server-only/envelope-item",
-                    "packages/trpc/server/envelope-router",
-                ],
+                {"email", "emails", "mailer", "notification", "notifications"},
+                ["email", "mailer", "notifications", "templates"],
             ),
             (
-                {"document", "documents"},
-                [
-                    "packages/lib/server-only/document",
-                    "packages/lib/server-only/document-data",
-                    "packages/trpc/server/document-router",
-                ],
+                {"upload", "storage", "s3", "file", "files", "attachment", "blob"},
+                ["storage", "upload", "uploads", "files", "attachments"],
             ),
             (
-                {"pdf", "storage", "upload", "s3"},
-                [
-                    "packages/lib/server-only/pdf",
-                    "packages/lib/server-only/document-data",
-                    "packages/lib/universal/upload",
-                    "apps/remix/server/api/files",
-                ],
+                {"cli", "command line", "command-line"},
+                ["cli", "commands", "bin"],
             ),
             (
-                {"api v1", "/api/v1", "ts-rest", "ts rest"},
-                [
-                    "packages/api",
-                    "packages/api/v1",
-                    "packages/api/hono.ts",
-                ],
+                {"ui", "frontend", "component", "components", "page", "pages", "view", "views"},
+                ["components", "pages", "views", "ui", "frontend", "client", "app"],
             ),
             (
-                {"api v2", "/api/v2", "openapi", "trpc-to-openapi"},
-                [
-                    "packages/trpc/server",
-                    "packages/trpc/server/open-api.ts",
-                    "apps/remix/server/router.ts",
-                    "apps/remix/server/trpc",
-                ],
+                {"backend", "server", "service", "services"},
+                ["server", "backend", "services", "api"],
             ),
             (
-                {"trpc", "frontend", "backend", "/api/trpc", "internal api"},
-                [
-                    "packages/trpc",
-                    "packages/trpc/react",
-                    "packages/trpc/client",
-                    "packages/trpc/server/context.ts",
-                    "apps/remix/server/trpc",
-                    "apps/remix/server/router.ts",
-                ],
-            ),
-            (
-                {"auth", "authentication", "session", "api token", "authorization", "bearer"},
-                [
-                    "packages/auth",
-                    "packages/lib/server-only/auth",
-                    "packages/lib/server-only/public-api",
-                    "packages/trpc/server/context.ts",
-                    "packages/trpc/server/trpc.ts",
-                    "packages/api/v1/middleware/authenticated.ts",
-                    "apps/remix/server/context.ts",
-                ],
+                {"middleware", "interceptor"},
+                ["middleware", "interceptors"],
             ),
             (
-                {"database", "postgres", "postgresql", "prisma", "kysely", "migration"},
-                [
-                    "packages/prisma",
-                    "packages/prisma/schema.prisma",
-                    "packages/prisma/migrations",
-                    ".env.example",
-                ],
+                {"docker", "container", "deployment", "deploy", "kubernetes", "helm", "ci", "cd", "pipeline"},
+                [".github/workflows", "docker", "dockerfile", "docker-compose", "helm", "deploy", "deployment", "ci"],
             ),
             (
-                {"remix", "hono", "react router", "route", "routes", "user interface"},
-                [
-                    "apps/remix/server",
-                    "apps/remix/app/routes",
-                    "apps/remix/app/root.tsx",
-                    "apps/remix/app/routes.ts",
-                ],
+                {"docs", "documentation", "readme"},
+                ["readme.md", "docs", "documentation"],
             ),
             (
-                {"test", "tests", "e2e", "playwright", "spec", "vitest"},
-                [
-                    "packages/app-tests",
-                    "packages/lib/vitest.config.ts",
-                    "packages/lib/package.json",
-                ],
+                {"util", "utils", "utility", "helper", "helpers", "common", "shared"},
+                ["utils", "util", "helpers", "common", "shared", "lib"],
             ),
             (
-                {
-                    "config",
-                    "configuration",
-                    "env",
-                    "environment",
-                    "local development",
-                    "self-host",
-                    "self hosting",
-                    "workspace",
-                    "workspaces",
-                    "turborepo",
-                    "turbo",
-                },
-                [
-                    ".env.example",
-                    "README.md",
-                    "package.json",
-                    "turbo.json",
-                    "apps/docs/content/docs/developers/local-development",
-                    "apps/docs/content/docs/self-hosting/configuration",
-                ],
+                {"type", "types", "interface", "schema"},
+                ["types", "type", "interfaces", "schemas"],
             ),
         ]
 

src/repo_fetcher.py

@@ -20,6 +20,16 @@ SUPPORTED_EXTENSIONS = {
     ".java",
     ".go",
     ".rs",
+    ".c",
+    ".h",
+    ".cc",
+    ".cpp",
+    ".cxx",
+    ".hpp",
+    ".hh",
+    ".y",
+    ".test",
+    ".tcl",
     ".md",
     ".mdx",
     ".json",
@@ -32,11 +42,20 @@ SUPPORTED_EXTENSIONS = {
     ".prisma",
 }
 
+# Extensionless / templated files that matter for a repo even though their
+# suffix (".in", no suffix, etc.) isn't a language extension on its own.
 SUPPORTED_FILENAMES = {
     ".env.example",
     "Dockerfile",
 }
 
+# Suffixes matched in addition to SUPPORTED_EXTENSIONS, for files like
+# "sqlite.h.in" or "Makefile.in" where the *last* suffix (".in") is a
+# build-template marker rather than the real language.
+SUPPORTED_TEMPLATE_SUFFIXES = {
+    ".in",
+}
+
 IGNORED_FILENAMES = {
     "package-lock.json",
     "yarn.lock",
@@ -68,7 +87,7 @@ IGNORED_DIRS = {
     "__pycache__",
 }
 
-MAX_FILE_SIZE_BYTES = 250_000
+MAX_FILE_SIZE_BYTES = 400_000
 
 
 class RepoFetcher:
@@ -181,9 +200,10 @@ class RepoFetcher:
                 continue
             if (
                 file_path.suffix.lower() not in SUPPORTED_EXTENSIONS
+                and file_path.suffix.lower() not in SUPPORTED_TEMPLATE_SUFFIXES
                 and file_path.name not in SUPPORTED_FILENAMES
             ):
                 continue
             if file_path.stat().st_size > MAX_FILE_SIZE_BYTES:
                 continue
-            yield file_path
+            yield file_path