diff --git a/scripts/run_validation.py b/scripts/run_validation.py new file mode 100644 index 0000000..db71fcf --- /dev/null +++ b/scripts/run_validation.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +""" +Run the full search-quality validation pipeline in one command. + +Steps: + 1. seed — scroll Qdrant, pick diverse multi-word symbols, write + identifier test cases to a temporary JSON file + 2. hybrid — validate BM25 + dense + RRF beats dense-only on identifier + queries (exact / tokenized / snake_case / prefix) + 3. hybrid-sem — run the same 30 curated semantic queries through hybrid vs + dense to check for regressions (test_cases_semantic.json) + 4. dense — validate dense-only semantic search quality (MRR / Hit@K) + +Overall exit code is 0 only when all three validators pass. + +Usage: + uv run scripts/run_validation.py + uv run scripts/run_validation.py --per-bucket 10 --limit 20 + uv run scripts/run_validation.py --url http://localhost:6333 \\ + --embeddings-url http://localhost:8087 +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import tempfile + +SCRIPTS = { + "seed": "scripts/seed_test_cases.py", + "hybrid": "scripts/validate_hybrid.py", + "dense": "scripts/validate_dense.py", +} + +SEMANTIC_CASES_FILE = "scripts/test_cases_semantic.json" + +SEPARATOR = "=" * 72 + + +def run(label: str, cmd: list[str]) -> int: + print(f"\n{SEPARATOR}", flush=True) + print(f" {label}", flush=True) + print(SEPARATOR, flush=True) + result = subprocess.run(cmd) + return result.returncode + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--url", + default=os.getenv("QDRANT_URL", "http://localhost:6333"), + help="Qdrant URL (default: $QDRANT_URL or http://localhost:6333)", + ) + parser.add_argument( + "--collection", + default=os.getenv("QDRANT_COLLECTION", "code_symbols"), + help="Collection name (default: $QDRANT_COLLECTION or code_symbols)", + ) + parser.add_argument( + "--embeddings-url", + default=os.getenv("EMBEDDINGS_URL", "http://localhost:8087"), + help="Jina TEI server URL (default: $EMBEDDINGS_URL or http://localhost:8087)", + ) + parser.add_argument( + "--per-bucket", + type=int, + default=10, + metavar="N", + help="Symbols to pick per symbol type during seeding (default: 10)", + ) + parser.add_argument( + "--scan-limit", + type=int, + default=500, + metavar="N", + help="Max points to scan per symbol type during seeding (default: 500)", + ) + parser.add_argument( + "--limit", + type=int, + default=10, + metavar="N", + help="Search result limit passed to both validators (default: 10)", + ) + parser.add_argument( + "--mrr-threshold", + type=float, + default=0.5, + metavar="F", + help="Minimum MRR for the dense validator to pass (default: 0.5)", + ) + parser.add_argument( + "--hit-threshold", + type=float, + default=0.8, + metavar="F", + help="Minimum Hit@10 for the dense validator to pass (default: 0.8)", + ) + args = parser.parse_args() + + py = sys.executable + qdrant_args = ["--url", args.url, "--collection", args.collection] + embed_args = ["--embeddings-url", args.embeddings_url] + + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp: + cases_file = tmp.name + + try: + # Step 1 — seed + seed_rc = run("STEP 1 / 3 — seed_test_cases.py", [ + py, SCRIPTS["seed"], + *qdrant_args, + "--per-bucket", str(args.per_bucket), + "--scan-limit", str(args.scan_limit), + "--output-json", cases_file, + ]) + if seed_rc != 0: + print("\n[ABORT] Seeding failed — check Qdrant connection and collection.", file=sys.stderr) + sys.exit(seed_rc) + + # Step 2 — hybrid validator (identifier kinds from seed) + hybrid_rc = run("STEP 2 / 4 — validate_hybrid.py (identifier queries: hybrid vs dense)", [ + py, SCRIPTS["hybrid"], + *qdrant_args, + *embed_args, + "--limit", str(args.limit), + "--test-cases-file", cases_file, + ]) + + # Step 3 — hybrid validator (semantic queries from curated file) + hybrid_sem_rc = run("STEP 3 / 4 — validate_hybrid.py (semantic queries: hybrid vs dense)", [ + py, SCRIPTS["hybrid"], + *qdrant_args, + *embed_args, + "--limit", str(args.limit), + "--test-cases-file", SEMANTIC_CASES_FILE, + "--kinds", "semantic", + ]) + + # Step 4 — dense validator (absolute quality on semantic queries) + dense_rc = run("STEP 4 / 4 — validate_dense.py (dense semantic search quality)", [ + py, SCRIPTS["dense"], + *qdrant_args, + *embed_args, + "--limit", str(args.limit), + "--mrr-threshold", str(args.mrr_threshold), + "--hit-threshold", str(args.hit_threshold), + ]) + + finally: + os.unlink(cases_file) + + # Combined summary + print(f"\n{SEPARATOR}") + print(" SUMMARY") + print(SEPARATOR) + print(f" validate_hybrid (identifiers) : {'PASS' if hybrid_rc == 0 else 'FAIL'}") + print(f" validate_hybrid (semantic) : {'PASS' if hybrid_sem_rc == 0 else 'FAIL'}") + print(f" validate_dense (semantic) : {'PASS' if dense_rc == 0 else 'FAIL'}") + overall = hybrid_rc == 0 and hybrid_sem_rc == 0 and dense_rc == 0 + print(f"\n Overall : {'PASS' if overall else 'FAIL'}") + print(SEPARATOR) + + sys.exit(0 if overall else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_test_cases.py b/scripts/seed_test_cases.py new file mode 100644 index 0000000..c229272 --- /dev/null +++ b/scripts/seed_test_cases.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +""" +Seed TEST_CASES for validate_hybrid.py / validate_dense.py from real Qdrant data. + +Scrolls the code_symbols collection, picks multi-word identifiers across +all meaningful symbol types, and prints a ready-to-paste TEST_CASES list +with up to five query styles per symbol: + + 1. exact -- the identifier as written (e.g. "PlaceOrderRequest") + 2. tokenized -- split into lowercase words (e.g. "place order request") + 3. snake_case -- tokenized words joined with _ (e.g. "place_order_request") + 4. prefix -- first two tokenized words only (e.g. "place order") + 5. semantic -- first sentence of docstring (skipped when no docstring) + +Review semantic entries before pasting: remove any whose docstring sentence +does not clearly describe the symbol on its own. + +Usage: + uv run scripts/seed_test_cases.py + uv run scripts/seed_test_cases.py --per-bucket 5 --scan-limit 1000 + uv run scripts/seed_test_cases.py --url http://localhost:6333 --collection code_symbols + uv run scripts/seed_test_cases.py --kinds semantic # for validate_dense.py + uv run scripts/seed_test_cases.py --kinds exact tokenized # identifier-only subset +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +from dataclasses import dataclass + +from qdrant_client import AsyncQdrantClient +from qdrant_client.models import FieldCondition, Filter, MatchValue + +# All symbol types that produce multi-word identifiers worth testing. +# Ordered so the most common ones come first in the output. +SYMBOL_TYPES = [ + "class", + "method", + "function", + "interface", + "constructor", + "enum", + "record", + "pydantic_model", + "dataclass", + "react_component", + "react_hook", +] + +DEFAULT_PER_BUCKET = 10 +DEFAULT_SCAN_LIMIT = 500 + + +@dataclass +class Candidate: + symbol_name: str + symbol_type: str + docstring: str | None + + +# --------------------------------------------------------------------------- +# Text helpers +# --------------------------------------------------------------------------- + +def is_multi_word(name: str) -> bool: + """True when the identifier has at least two word components.""" + has_camel = bool(re.search(r"[a-z][A-Z]", name)) + has_pascal = bool(re.search(r"^[A-Z][a-z]+[A-Z]", name)) + has_snake = "_" in name and len(name.split("_")) >= 2 + return has_camel or has_pascal or has_snake + + +def tokenize(name: str) -> str: + """Split camelCase / PascalCase / snake_case into lowercase words.""" + s = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", name) + s = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1 \2", s) + s = s.replace("_", " ").replace("-", " ") + return s.lower().strip() + + +def to_snake_case(name: str) -> str: + """Convert any identifier to snake_case.""" + return tokenize(name).replace(" ", "_") + + +def prefix_query(name: str, words: int = 2) -> str | None: + """First `words` words of the tokenized name, or None if too few words.""" + parts = tokenize(name).split() + if len(parts) <= words: + return None + return " ".join(parts[:words]) + + +def first_sentence(docstring: str) -> str | None: + """Return the first useful sentence from a docstring, or None.""" + cleaned = re.sub(r"^(\"\"\"|'''|/\*\*?|//)\s*", "", docstring.strip()) + cleaned = re.sub(r'\s*(\"\"\"|\'\'\'|\*/)$', "", cleaned) + # Collapse javadoc/jsdoc continuation lines (" * text" → " text") + cleaned = re.sub(r"\n\s*\*\s?", " ", cleaned).strip() + # Drop @param / @return / @throws tags — not useful as queries + cleaned = re.sub(r"@\w+[^\n]*", "", cleaned).strip() + sentence = re.split(r"(?<=[.!?])\s", cleaned)[0].strip().rstrip(".") + if len(sentence) < 15: + return None + return sentence + + +# --------------------------------------------------------------------------- +# Test-case generation +# --------------------------------------------------------------------------- + +def make_test_cases(c: Candidate) -> list[tuple[str, str, str]]: + """Return (query, expected_symbol_name, kind) triples for one candidate.""" + cases: list[tuple[str, str, str]] = [] + tokens = tokenize(c.symbol_name) + + # 1. exact + cases.append((c.symbol_name, c.symbol_name, "exact")) + + # 2. tokenized — only if it differs from the raw name + if tokens != c.symbol_name.lower(): + cases.append((tokens, c.symbol_name, "tokenized")) + + # 3. snake_case — only if it differs from both the raw name and tokenized + snake = to_snake_case(c.symbol_name) + if snake != c.symbol_name and snake != c.symbol_name.lower(): + cases.append((snake, c.symbol_name, "snake_case")) + + # 4. prefix — only when the name has more than two words + prefix = prefix_query(c.symbol_name) + if prefix: + cases.append((prefix, c.symbol_name, "prefix")) + + # 5. semantic — first sentence of docstring when present + if c.docstring: + sentence = first_sentence(c.docstring) + if sentence: + cases.append((sentence, c.symbol_name, "semantic")) + + return cases + + +# --------------------------------------------------------------------------- +# Qdrant sampling +# --------------------------------------------------------------------------- + +async def scroll_bucket( + client: AsyncQdrantClient, + collection: str, + symbol_type: str, + scan_limit: int, +) -> list[Candidate]: + candidates: list[Candidate] = [] + offset = None + fetched = 0 + + while fetched < scan_limit: + batch_size = min(200, scan_limit - fetched) + results, offset = await client.scroll( + collection_name=collection, + scroll_filter=Filter( + must=[FieldCondition(key="symbol_type", match=MatchValue(value=symbol_type))] + ), + limit=batch_size, + offset=offset, + with_payload=["symbol_name", "symbol_type", "docstring"], + with_vectors=False, + ) + for point in results: + name = (point.payload.get("symbol_name") or "").strip() + if name and is_multi_word(name): + candidates.append( + Candidate( + symbol_name=name, + symbol_type=symbol_type, + docstring=point.payload.get("docstring"), + ) + ) + fetched += len(results) + if offset is None: + break + + return candidates + + +def pick_diverse(candidates: list[Candidate], n: int) -> list[Candidate]: + """ + Pick n candidates with name diversity: + - Deduplicate by exact symbol_name. + - Group by the first tokenized word; take at most ceil(n/groups) per group + so we don't end up with e.g. five "get*" methods and nothing else. + - Within each group prefer longer names (more tokenization surface). + """ + seen_names: set[str] = set() + by_first_word: dict[str, list[Candidate]] = {} + + for c in candidates: + if c.symbol_name in seen_names: + continue + seen_names.add(c.symbol_name) + first_word = tokenize(c.symbol_name).split()[0] + by_first_word.setdefault(first_word, []).append(c) + + # Sort within each group by name length descending + for group in by_first_word.values(): + group.sort(key=lambda c: len(c.symbol_name), reverse=True) + + # Round-robin across groups until we have n candidates + picked: list[Candidate] = [] + groups = list(by_first_word.values()) + i = 0 + while len(picked) < n and groups: + idx = i % len(groups) + group = groups[idx] + if group: + picked.append(group.pop(0)) + if not group: + groups.pop(idx) + i = max(0, i - 1) + else: + i += 1 + else: + groups.pop(idx) + + return picked + + +# --------------------------------------------------------------------------- +# Output rendering +# --------------------------------------------------------------------------- + +def render(all_cases: list[tuple[str, str, str, str]]) -> str: + """Render a ready-to-paste TEST_CASES block.""" + lines = ["TEST_CASES = ["] + current_symbol = None + + for query, expected, kind, symbol_type in all_cases: + if expected != current_symbol: + if current_symbol is not None: + lines.append("") + lines.append(f" # {symbol_type}: {expected}") + current_symbol = expected + + padding = " " * max(1, 52 - len(repr(query))) + lines.append(f" ({query!r},{padding}{expected!r}, {kind!r}),") + + lines.append("]") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +async def main( + url: str, + collection: str, + per_bucket: int, + scan_limit: int, + kinds: set[str] | None, + output_json: str | None, +) -> None: + client = AsyncQdrantClient(url=url) + try: + info = await client.get_collection(collection) + print(f"# Collection '{collection}' — {info.points_count} points total", file=sys.stderr) + + # (query, expected_name, kind, symbol_type) + all_cases: list[tuple[str, str, str, str]] = [] + + for symbol_type in SYMBOL_TYPES: + candidates = await scroll_bucket(client, collection, symbol_type, scan_limit) + picked = pick_diverse(candidates, per_bucket) + print( + f"# {symbol_type}: scanned up to {scan_limit}, " + f"found {len(candidates)} multi-word, picked {len(picked)}", + file=sys.stderr, + ) + for c in picked: + for query, expected, kind in make_test_cases(c): + if kinds is None or kind in kinds: + all_cases.append((query, expected, kind, symbol_type)) + + if not all_cases: + print( + "# No multi-word identifiers found — is the collection indexed?", + file=sys.stderr, + ) + sys.exit(1) + + if output_json: + with open(output_json, "w") as f: + json.dump([[q, e, k] for q, e, k, _ in all_cases], f) + print(f"# Wrote {len(all_cases)} test cases to {output_json}", file=sys.stderr) + else: + print() + print(render(all_cases)) + + finally: + await client.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--url", + default=os.getenv("QDRANT_URL", "http://localhost:6333"), + help="Qdrant URL (default: $QDRANT_URL or http://localhost:6333)", + ) + parser.add_argument( + "--collection", + default=os.getenv("QDRANT_COLLECTION", "code_symbols"), + help="Collection name (default: $QDRANT_COLLECTION or code_symbols)", + ) + parser.add_argument( + "--per-bucket", + type=int, + default=DEFAULT_PER_BUCKET, + metavar="N", + help=f"Symbols to pick per symbol type (default: {DEFAULT_PER_BUCKET})", + ) + parser.add_argument( + "--scan-limit", + type=int, + default=DEFAULT_SCAN_LIMIT, + metavar="N", + help=f"Max points to scan per symbol type (default: {DEFAULT_SCAN_LIMIT})", + ) + parser.add_argument( + "--kinds", + nargs="+", + choices=["exact", "tokenized", "snake_case", "prefix", "semantic"], + default=None, + metavar="KIND", + help="Only emit test cases of these kinds (default: all). " + "Use '--kinds semantic' for validate_dense.py.", + ) + parser.add_argument( + "--output-json", + default=None, + metavar="FILE", + help="Write test cases to a JSON file instead of printing Python code. " + "Used by run_validation.py.", + ) + args = parser.parse_args() + asyncio.run( + main(args.url, args.collection, args.per_bucket, args.scan_limit, + set(args.kinds) if args.kinds else None, + args.output_json) + ) diff --git a/scripts/test_cases_semantic.json b/scripts/test_cases_semantic.json new file mode 100644 index 0000000..590ca0d --- /dev/null +++ b/scripts/test_cases_semantic.json @@ -0,0 +1,252 @@ +[ + [ + "BeginRegistration POST /api/register/begin", + "BeginRegistration", + "semantic" + ], + [ + "FinishRegistration POST /api/register/finish", + "FinishRegistration", + "semantic" + ], + [ + "BeginLogin POST /api/authenticate/begin", + "BeginLogin", + "semantic" + ], + [ + "FinishLogin POST /api/authenticate/finish", + "FinishLogin", + "semantic" + ], + [ + "GetRegisteredPasskeys GET /api/users/:username/registered-passkeys", + "GetRegisteredPasskeys", + "semantic" + ], + [ + "Spring Security web security filter chain setup", + "SecurityConfiguration", + "semantic" + ], + [ + "OAuth2 authorization server beans and configuration", + "AuthorizationServerConfiguration", + "semantic" + ], + [ + "JPA service for persisting OAuth2 authorization consents", + "JpaAuthorizationConsentService", + "semantic" + ], + [ + "JPA-backed storage and retrieval of OAuth2 authorizations", + "JpaAuthorizationService", + "semantic" + ], + [ + "authentication provider that checks for leaked or breached passwords", + "LeakedPasswordsAuthenticationProvider", + "semantic" + ], + [ + "authentication provider for WebAuthn passkey login", + "WebAuthnAuthenticationProvider", + "semantic" + ], + [ + "controller for WebAuthn passkey registration and authentication", + "WebAuthnController", + "semantic" + ], + [ + "initializes OAuth2 registered clients at application startup", + "RegisteredClientInitializer", + "semantic" + ], + [ + "seeds initial user accounts in the database on startup", + "UserInitializer", + "semantic" + ], + [ + "JPA implementation of the OAuth2 registered client repository", + "JpaClientRepository", + "semantic" + ], + [ + "exception thrown when WebAuthn operation fails", + "WebAuthnException", + "semantic" + ], + [ + "Spring Cloud Gateway route definitions with token relay", + "RouteConfiguration", + "semantic" + ], + [ + "bean that customizes JWT token claims before issuing", + "tokenCustomizer", + "semantic" + ], + [ + "find an OAuth2 authorization by any token or code value", + "findByStateOrAuthorizationCodeValueOrAccessTokenValueOrRefreshTokenValueOrOidcIdTokenValueOrUserCodeValueOrDeviceCodeValue", + "semantic" + ], + [ + "fetch a user record by their username", + "GetUserByUsername", + "semantic" + ], + [ + "retrieve a user together with their associated passkey credentials", + "GetUserWithCredentials", + "semantic" + ], + [ + "initialize the WebAuthn relying party configuration", + "InitWebAuthn", + "semantic" + ], + [ + "HTTP handler to check if a password appears in breach data", + "CheckPasswordHandler", + "semantic" + ], + [ + "HTTP handler to look up a password hash in the leaked passwords store", + "GetByHashHandler", + "semantic" + ], + [ + "open the database connection for the service", + "ConnectDatabase", + "semantic" + ], + [ + "JPA repository for managing authorization consent records", + "AuthorizationConsentRepository", + "semantic" + ], + [ + "repository interface for looking up registered OAuth2 clients", + "ClientRepository", + "semantic" + ], + [ + "client interface for calling the leaked passwords API", + "LeakedPasswordsClient", + "semantic" + ], + [ + "request body for checking if a password has been compromised", + "CheckPasswordRequest", + "semantic" + ], + [ + "response indicating whether a password hash was found in breach data", + "CheckPasswordResponse", + "semantic" + ], + [ + "security configuration", + "SecurityConfiguration", + "semantic" + ], + [ + "service that checks whether a password has been exposed in a breach", + "LeakedPasswordsService", + "semantic" + ], + [ + "repository for storing authorization records", + "AuthorizationRepository", + "semantic" + ], + [ + "authentication provider for hardware security key login", + "WebAuthnAuthenticationProvider", + "semantic" + ], + [ + "persists OAuth2 tokens, codes and authorization state", + "JpaAuthorizationService", + "semantic" + ], + [ + "HTTP client interface for the breached passwords backend", + "LeakedPasswordsClient", + "semantic" + ], + [ + "starts a passkey authentication ceremony", + "BeginLogin", + "semantic" + ], + [ + "error raised during a passkey or WebAuthn operation", + "WebAuthnException", + "semantic" + ], + [ + "gateway route definitions", + "RouteConfiguration", + "semantic" + ], + [ + "creates default application users on startup", + "UserInitializer", + "semantic" + ], + [ + "find authorization by token value", + "findByStateOrAuthorizationCodeValueOrAccessTokenValueOrRefreshTokenValueOrOidcIdTokenValueOrUserCodeValueOrDeviceCodeValue", + "semantic" + ], + [ + "manages user consent grants for OAuth2 scopes", + "JpaAuthorizationConsentService", + "semantic" + ], + [ + "represents a registered passkey device credential", + "RegisteredPasskey", + "semantic" + ], + [ + "connect to the database", + "ConnectDatabase", + "semantic" + ], + [ + "endpoint to begin passkey registration", + "BeginRegistration", + "semantic" + ], + [ + "validates identity using a hardware authenticator", + "WebAuthnAuthenticationProvider", + "semantic" + ], + [ + "exposes endpoints for querying the leaked passwords dataset", + "LeakedPasswordsApi", + "semantic" + ], + [ + "pre-loads OAuth2 clients into the authorization server", + "RegisteredClientInitializer", + "semantic" + ], + [ + "retrieves and deserializes a stored WebAuthn session", + "getAndParseWebAuthnSession", + "semantic" + ], + [ + "adds custom claims to JWT access tokens", + "JwtTokenCustomizerConfig", + "semantic" + ] +] \ No newline at end of file diff --git a/scripts/validate_dense.py b/scripts/validate_dense.py new file mode 100644 index 0000000..138875c --- /dev/null +++ b/scripts/validate_dense.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +""" +Validate dense (embedding-only) semantic search quality. + +Runs every entry in TEST_CASES through dense vector search and measures: + - Hit@1, Hit@3, Hit@5, Hit@10 (% of queries where expected symbol ranks ≤K) + - MRR (Mean Reciprocal Rank: mean of 1/rank, higher is better) + - Average rank + +Success criteria (configurable via --mrr-threshold / --hit-threshold): + MRR ≥ 0.5 — expected symbol ranks roughly 2nd on average + Hit@10 ≥ 0.8 — expected symbol found in top-10 for 80% of queries + +Populate TEST_CASES from semantic entries only: + uv run scripts/seed_test_cases.py --kinds semantic --per-bucket 10 + # paste the output block below, replacing the empty list + +Usage: + uv run scripts/validate_dense.py + uv run scripts/validate_dense.py --embeddings-url http://localhost:8087 + uv run scripts/validate_dense.py --limit 10 --mrr-threshold 0.6 --hit-threshold 0.9 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from dataclasses import dataclass + +import httpx +from qdrant_client import AsyncQdrantClient + +# --------------------------------------------------------------------------- +# TEST_CASES — populate via: uv run scripts/seed_test_cases.py --kinds semantic +# Each entry: (query, expected_symbol_name, kind) +# --------------------------------------------------------------------------- + +TEST_CASES: list[tuple[str, str, str]] = [ + # From actual docstrings + ("BeginRegistration POST /api/register/begin", "BeginRegistration", "semantic"), + ("FinishRegistration POST /api/register/finish", "FinishRegistration", "semantic"), + ("BeginLogin POST /api/authenticate/begin", "BeginLogin", "semantic"), + ("FinishLogin POST /api/authenticate/finish", "FinishLogin", "semantic"), + ("GetRegisteredPasskeys GET /api/users/:username/registered-passkeys", "GetRegisteredPasskeys", "semantic"), + + # Manual — natural language queries + ("Spring Security web security filter chain setup", "SecurityConfiguration", "semantic"), + ("OAuth2 authorization server beans and configuration", "AuthorizationServerConfiguration", "semantic"), + ("JPA service for persisting OAuth2 authorization consents", "JpaAuthorizationConsentService", "semantic"), + ("JPA-backed storage and retrieval of OAuth2 authorizations", "JpaAuthorizationService", "semantic"), + ("authentication provider that checks for leaked or breached passwords", "LeakedPasswordsAuthenticationProvider", "semantic"), + ("authentication provider for WebAuthn passkey login", "WebAuthnAuthenticationProvider", "semantic"), + ("controller for WebAuthn passkey registration and authentication", "WebAuthnController", "semantic"), + ("initializes OAuth2 registered clients at application startup", "RegisteredClientInitializer", "semantic"), + ("seeds initial user accounts in the database on startup", "UserInitializer", "semantic"), + ("JPA implementation of the OAuth2 registered client repository", "JpaClientRepository", "semantic"), + ("exception thrown when WebAuthn operation fails", "WebAuthnException", "semantic"), + ("Spring Cloud Gateway route definitions with token relay", "RouteConfiguration", "semantic"), + ("bean that customizes JWT token claims before issuing", "tokenCustomizer", "semantic"), + ("find an OAuth2 authorization by any token or code value", "findByStateOrAuthorizationCodeValueOrAccessTokenValueOrRefreshTokenValueOrOidcIdTokenValueOrUserCodeValueOrDeviceCodeValue", "semantic"), + ("fetch a user record by their username", "GetUserByUsername", "semantic"), + ("retrieve a user together with their associated passkey credentials", "GetUserWithCredentials", "semantic"), + ("initialize the WebAuthn relying party configuration", "InitWebAuthn", "semantic"), + ("HTTP handler to check if a password appears in breach data", "CheckPasswordHandler", "semantic"), + ("HTTP handler to look up a password hash in the leaked passwords store","GetByHashHandler", "semantic"), + ("open the database connection for the service", "ConnectDatabase", "semantic"), + ("JPA repository for managing authorization consent records", "AuthorizationConsentRepository", "semantic"), + ("repository interface for looking up registered OAuth2 clients", "ClientRepository", "semantic"), + ("client interface for calling the leaked passwords API", "LeakedPasswordsClient", "semantic"), + ("request body for checking if a password has been compromised", "CheckPasswordRequest", "semantic"), + ("response indicating whether a password hash was found in breach data", "CheckPasswordResponse", "semantic"), + + # Ambiguous — query could reasonably match several symbols + ("security configuration", "SecurityConfiguration", "semantic"), + ("service that checks whether a password has been exposed in a breach", "LeakedPasswordsService", "semantic"), + ("repository for storing authorization records", "AuthorizationRepository", "semantic"), + ("authentication provider for hardware security key login", "WebAuthnAuthenticationProvider", "semantic"), + ("persists OAuth2 tokens, codes and authorization state", "JpaAuthorizationService", "semantic"), + ("HTTP client interface for the breached passwords backend", "LeakedPasswordsClient", "semantic"), + ("starts a passkey authentication ceremony", "BeginLogin", "semantic"), + ("error raised during a passkey or WebAuthn operation", "WebAuthnException", "semantic"), + ("gateway route definitions", "RouteConfiguration", "semantic"), + ("creates default application users on startup", "UserInitializer", "semantic"), + ("find authorization by token value", "findByStateOrAuthorizationCodeValueOrAccessTokenValueOrRefreshTokenValueOrOidcIdTokenValueOrUserCodeValueOrDeviceCodeValue", "semantic"), + ("manages user consent grants for OAuth2 scopes", "JpaAuthorizationConsentService", "semantic"), + ("represents a registered passkey device credential", "RegisteredPasskey", "semantic"), + ("connect to the database", "ConnectDatabase", "semantic"), + ("endpoint to begin passkey registration", "BeginRegistration", "semantic"), + ("validates identity using a hardware authenticator", "WebAuthnAuthenticationProvider", "semantic"), + ("exposes endpoints for querying the leaked passwords dataset", "LeakedPasswordsApi", "semantic"), + ("pre-loads OAuth2 clients into the authorization server", "RegisteredClientInitializer", "semantic"), + ("retrieves and deserializes a stored WebAuthn session", "getAndParseWebAuthnSession", "semantic"), + ("adds custom claims to JWT access tokens", "JwtTokenCustomizerConfig", "semantic"), +] + +DEFAULT_LIMIT = 10 +DEFAULT_MRR_THRESHOLD = 0.5 +DEFAULT_HIT_THRESHOLD = 0.8 +HIT_AT_KS = [1, 3, 5, 10] + + +# --------------------------------------------------------------------------- +# Dense embedding helper (mirrors server/embeddings/jina.py) +# --------------------------------------------------------------------------- + +async def get_dense_vector(client: httpx.AsyncClient, embeddings_url: str, text: str) -> list[float]: + response = await client.post( + f"{embeddings_url}/embed", + json={"inputs": [text]}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + if isinstance(data, list): + return data[0] + return data["data"][0]["embedding"] + + +# --------------------------------------------------------------------------- +# Search helper +# --------------------------------------------------------------------------- + +async def search_dense( + client: AsyncQdrantClient, + collection: str, + dense: list[float], + limit: int, +) -> list[str]: + result = await client.query_points( + collection_name=collection, + query=dense, + using="text-dense", + limit=limit, + with_payload=["symbol_name"], + ) + return [p.payload.get("symbol_name", "") for p in result.points] + + +def get_rank(names: list[str], expected: str, fallback: int) -> int: + try: + return names.index(expected) + 1 + except ValueError: + return fallback + + +# --------------------------------------------------------------------------- +# Report +# --------------------------------------------------------------------------- + +@dataclass +class Result: + query: str + expected: str + rank: int + limit: int + + @property + def found(self) -> bool: + return self.rank <= self.limit + + def hit_at(self, k: int) -> bool: + return self.rank <= k + + @property + def reciprocal_rank(self) -> float: + return 1.0 / self.rank if self.found else 0.0 + + +def print_report(results: list[Result], limit: int, mrr_threshold: float, hit_threshold: float) -> bool: + col_q = min(max(len(r.query) for r in results), 52) + col_e = min(max(len(r.expected) for r in results), 36) + + header = f"{'Query':<{col_q}} {'Expected':<{col_e}} {'Rank':>6} {'RR':>6}" + print(header) + print("-" * len(header)) + + for r in results: + q = r.query if len(r.query) <= col_q else r.query[: col_q - 1] + "…" + e = r.expected if len(r.expected) <= col_e else r.expected[: col_e - 1] + "…" + rank_str = str(r.rank) if r.found else f">{limit}" + rr_str = f"{r.reciprocal_rank:.3f}" + print(f"{q:<{col_q}} {e:<{col_e}} {rank_str:>6} {rr_str:>6}") + + print() + + n = len(results) + mrr = sum(r.reciprocal_rank for r in results) / n + avg_rank = sum(r.rank for r in results) / n + + # Hit@K table + print(f"{'Metric':<12} {'Value':>8}") + print("-" * 24) + for k in HIT_AT_KS: + if k > limit: + continue + hit = sum(1 for r in results if r.hit_at(k)) / n + print(f"Hit@{k:<8} {hit:>7.1%}") + print(f"{'MRR':<12} {mrr:>8.4f}") + print(f"{'Avg rank':<12} {avg_rank:>8.2f}") + print() + + # Success criteria + hit10 = sum(1 for r in results if r.hit_at(min(10, limit))) / n + passed = True + + c1_pass = mrr >= mrr_threshold + status = "PASS" if c1_pass else "FAIL" + print(f"[{status}] CRITERION 1 — MRR: {mrr:.4f} (threshold: ≥{mrr_threshold})") + if not c1_pass: + passed = False + + effective_k = min(10, limit) + c2_pass = hit10 >= hit_threshold + status = "PASS" if c2_pass else "FAIL" + print(f"[{status}] CRITERION 2 — Hit@{effective_k}: {hit10:.1%} (threshold: ≥{hit_threshold:.0%})") + if not c2_pass: + passed = False + + return passed + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def _load_test_cases(path: str, kinds: set[str] | None) -> list[tuple[str, str, str]]: + with open(path) as f: + rows = json.load(f) + cases = [(q, e, k) for q, e, k in rows] + if kinds: + cases = [(q, e, k) for q, e, k in cases if k in kinds] + return cases + + +async def main( + url: str, + collection: str, + embeddings_url: str, + limit: int, + mrr_threshold: float, + hit_threshold: float, + test_cases_file: str | None, + kinds: set[str] | None, +) -> None: + cases: list[tuple[str, str, str]] = ( + _load_test_cases(test_cases_file, kinds) if test_cases_file else list(TEST_CASES) + ) + if kinds and not test_cases_file: + cases = [(q, e, k) for q, e, k in cases if k in kinds] + if not cases: + print( + "TEST_CASES is empty.\n" + "Run: uv run scripts/seed_test_cases.py --kinds semantic\n" + "then paste the output into this file, or use --test-cases-file.", + file=sys.stderr, + ) + sys.exit(1) + + qdrant = AsyncQdrantClient(url=url) + fallback = limit + 1 + + async with httpx.AsyncClient() as http: + results: list[Result] = [] + + for i, entry in enumerate(cases, 1): + query, expected = entry[0], entry[1] + print(f" [{i}/{len(TEST_CASES)}] {query!r}…", file=sys.stderr, end="\r") + dense = await get_dense_vector(http, embeddings_url, query) + names = await search_dense(qdrant, collection, dense, limit) + results.append(Result( + query=query, + expected=expected, + rank=get_rank(names, expected, fallback), + limit=limit, + )) + + print(" " * 60, file=sys.stderr, end="\r") + print(f"Dense semantic search — {len(results)} queries, search limit={limit}\n", flush=True) + passed = print_report(results, limit, mrr_threshold, hit_threshold) + + await qdrant.close() + sys.exit(0 if passed else 1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--url", + default=os.getenv("QDRANT_URL", "http://localhost:6333"), + help="Qdrant URL (default: $QDRANT_URL or http://localhost:6333)", + ) + parser.add_argument( + "--collection", + default=os.getenv("QDRANT_COLLECTION", "code_symbols"), + help="Collection name (default: $QDRANT_COLLECTION or code_symbols)", + ) + parser.add_argument( + "--embeddings-url", + default=os.getenv("EMBEDDINGS_URL", "http://localhost:8087"), + help="Jina TEI server URL (default: $EMBEDDINGS_URL or http://localhost:8087)", + ) + parser.add_argument( + "--limit", + type=int, + default=DEFAULT_LIMIT, + metavar="N", + help=f"Number of results to fetch per search (default: {DEFAULT_LIMIT})", + ) + parser.add_argument( + "--mrr-threshold", + type=float, + default=DEFAULT_MRR_THRESHOLD, + metavar="F", + help=f"Minimum MRR to pass criterion 1 (default: {DEFAULT_MRR_THRESHOLD})", + ) + parser.add_argument( + "--hit-threshold", + type=float, + default=DEFAULT_HIT_THRESHOLD, + metavar="F", + help=f"Minimum Hit@10 to pass criterion 2 (default: {DEFAULT_HIT_THRESHOLD})", + ) + parser.add_argument( + "--test-cases-file", + default=None, + metavar="FILE", + help="JSON file produced by seed_test_cases.py --output-json. " + "Overrides the hardcoded TEST_CASES list.", + ) + parser.add_argument( + "--kinds", + nargs="+", + choices=["exact", "tokenized", "snake_case", "prefix", "semantic"], + default=None, + metavar="KIND", + help="Only validate test cases of these kinds (default: all).", + ) + args = parser.parse_args() + asyncio.run(main( + args.url, args.collection, args.embeddings_url, + args.limit, args.mrr_threshold, args.hit_threshold, + args.test_cases_file, + set(args.kinds) if args.kinds else None, + )) diff --git a/scripts/validate_hybrid.py b/scripts/validate_hybrid.py new file mode 100644 index 0000000..18c7d7d --- /dev/null +++ b/scripts/validate_hybrid.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +""" +Validate that hybrid (BM25 + dense + RRF) search outperforms dense-only baseline. + +Runs every entry in TEST_CASES through both search modes, measures ranking +positions, and checks two success criteria: + + CRITERION 1 Exact / tokenized / snake_case / prefix queries rank on average + ≥2 positions higher (lower rank number) in hybrid mode. + CRITERION 2 No semantic query drops more than 1 position in hybrid mode + compared with dense-only. + +Populate TEST_CASES before running: + uv run scripts/seed_test_cases.py --per-bucket 5 + # paste the output block below, replacing the empty list + +Usage: + uv run scripts/validate_hybrid.py + uv run scripts/validate_hybrid.py --embeddings-url http://localhost:8087 + uv run scripts/validate_hybrid.py --limit 20 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +from dataclasses import dataclass + +import httpx +from fastembed.sparse.bm25 import Bm25 +from qdrant_client import AsyncQdrantClient +from qdrant_client.models import ( + Fusion, + FusionQuery, + Prefetch, + SparseVector, +) + +# --------------------------------------------------------------------------- +# TEST_CASES — populate via: uv run scripts/seed_test_cases.py +# Each entry: (query, expected_symbol_name, kind) +# kind: "exact" | "tokenized" | "snake_case" | "prefix" | "semantic" +# --------------------------------------------------------------------------- + +TEST_CASES: list[tuple[str, str, str]] = [ + # Paste seed_test_cases.py output here. + # Example: + # ("PlaceOrderRequest", "PlaceOrderRequest", "exact"), + # ("place order request","PlaceOrderRequest", "tokenized"), + # ("place order", "PlaceOrderRequest", "prefix"), +] + +# Kinds that should benefit from BM25 (identifier-based queries) +IDENTIFIER_KINDS = {"exact", "tokenized", "snake_case", "prefix"} +SEMANTIC_KIND = "semantic" + +DEFAULT_LIMIT = 10 + + +# --------------------------------------------------------------------------- +# BM25 helper (mirrors server/embeddings/bm25.py without importing server/) +# --------------------------------------------------------------------------- + +def _split_code_identifiers(text: str) -> str: + """Inline of server/embeddings/code_tokenizer.py:split_code_identifiers.""" + expanded = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", text) + expanded = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1 \2", expanded) + expanded = expanded.replace("_", " ").replace("-", " ") + return text + "\n" + expanded + + +async def get_sparse_vector(model: Bm25, text: str) -> SparseVector: + loop = asyncio.get_running_loop() + prepared = _split_code_identifiers(text) + [embedding] = await loop.run_in_executor(None, lambda: list(model.query_embed(prepared))) + return SparseVector(indices=embedding.indices.tolist(), values=embedding.values.tolist()) + + +# --------------------------------------------------------------------------- +# Dense embedding helper (mirrors server/embeddings/jina.py) +# --------------------------------------------------------------------------- + +async def get_dense_vector(client: httpx.AsyncClient, embeddings_url: str, text: str) -> list[float]: + response = await client.post( + f"{embeddings_url}/embed", + json={"inputs": [text]}, + timeout=30.0, + ) + response.raise_for_status() + data = response.json() + if isinstance(data, list): + return data[0] + return data["data"][0]["embedding"] + + +# --------------------------------------------------------------------------- +# Search helpers +# --------------------------------------------------------------------------- + +async def search_hybrid( + client: AsyncQdrantClient, + collection: str, + dense: list[float], + sparse: SparseVector, + limit: int, +) -> list[str]: + result = await client.query_points( + collection_name=collection, + prefetch=[ + Prefetch(query=dense, using="text-dense", limit=limit * 2), + Prefetch(query=sparse, using="text-sparse", limit=limit * 2), + ], + query=FusionQuery(fusion=Fusion.RRF), + limit=limit, + with_payload=["symbol_name"], + ) + return [p.payload.get("symbol_name", "") for p in result.points] + + +async def search_dense( + client: AsyncQdrantClient, + collection: str, + dense: list[float], + limit: int, +) -> list[str]: + result = await client.query_points( + collection_name=collection, + query=dense, + using="text-dense", + limit=limit, + with_payload=["symbol_name"], + ) + return [p.payload.get("symbol_name", "") for p in result.points] + + +def get_rank(names: list[str], expected: str, fallback: int) -> int: + try: + return names.index(expected) + 1 + except ValueError: + return fallback + + +# --------------------------------------------------------------------------- +# Report +# --------------------------------------------------------------------------- + +@dataclass +class Result: + query: str + expected: str + kind: str + dense_rank: int + hybrid_rank: int + + @property + def delta(self) -> int: + return self.dense_rank - self.hybrid_rank + + +def print_report(results: list[Result], limit: int) -> bool: + col_q = max(len(r.query) for r in results) + col_q = min(col_q, 48) + col_e = max(len(r.expected) for r in results) + col_e = min(col_e, 40) + + header = ( + f"{'Query':<{col_q}} {'Expected':<{col_e}} {'Kind':<10} " + f"{'Dense':>5} {'Hybrid':>6} {'Delta':>5}" + ) + print(header) + print("-" * len(header)) + + for r in results: + q = r.query if len(r.query) <= col_q else r.query[: col_q - 1] + "…" + e = r.expected if len(r.expected) <= col_e else r.expected[: col_e - 1] + "…" + d_str = str(r.dense_rank) if r.dense_rank <= limit else f">{limit}" + h_str = str(r.hybrid_rank) if r.hybrid_rank <= limit else f">{limit}" + sign = "+" if r.delta > 0 else "" + print( + f"{q:<{col_q}} {e:<{col_e}} {r.kind:<10} " + f"{d_str:>5} {h_str:>6} {sign}{r.delta:>4}" + ) + + print() + + # Per-kind summary + from collections import defaultdict + + by_kind: dict[str, list[int]] = defaultdict(list) + for r in results: + by_kind[r.kind].append(r.delta) + + print(f"{'Kind':<12} {'Count':>5} {'Avg delta':>9} {'Min delta':>9} {'Max delta':>9}") + print("-" * 52) + for kind in ["exact", "tokenized", "snake_case", "prefix", "semantic"]: + deltas = by_kind.get(kind, []) + if not deltas: + continue + avg = sum(deltas) / len(deltas) + print( + f"{kind:<12} {len(deltas):>5} {avg:>+9.2f} " + f"{min(deltas):>+9} {max(deltas):>+9}" + ) + print() + + # Success criteria + identifier_deltas = [r.delta for r in results if r.kind in IDENTIFIER_KINDS] + semantic_deltas = [r.delta for r in results if r.kind == SEMANTIC_KIND] + + passed = True + + if identifier_deltas: + avg_id = sum(identifier_deltas) / len(identifier_deltas) + c1_pass = avg_id >= 2.0 + status = "PASS" if c1_pass else "FAIL" + print(f"[{status}] CRITERION 1 — identifier avg delta: {avg_id:+.2f} (threshold: ≥+2.0)") + if not c1_pass: + passed = False + else: + print("[SKIP] CRITERION 1 — no identifier-kind test cases found") + + if semantic_deltas: + worst = min(semantic_deltas) + c2_pass = worst >= -1 + status = "PASS" if c2_pass else "FAIL" + print(f"[{status}] CRITERION 2 — semantic worst delta: {worst:+d} (threshold: ≥-1)") + if not c2_pass: + passed = False + else: + print("[SKIP] CRITERION 2 — no semantic-kind test cases found") + + return passed + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +def _load_test_cases(path: str) -> list[tuple[str, str, str]]: + with open(path) as f: + return [(q, e, k) for q, e, k in json.load(f)] + + +async def main( + url: str, + collection: str, + embeddings_url: str, + limit: int, + test_cases_file: str | None, + kinds: set[str] | None, +) -> None: + cases: list[tuple[str, str, str]] = ( + _load_test_cases(test_cases_file) if test_cases_file else list(TEST_CASES) + ) + if kinds: + cases = [(q, e, k) for q, e, k in cases if k in kinds] + if not cases: + print( + "TEST_CASES is empty.\n" + "Run: uv run scripts/seed_test_cases.py\n" + "then paste the output into this file, or use --test-cases-file.", + file=sys.stderr, + ) + sys.exit(1) + + print(f"Loading BM25 model…", file=sys.stderr) + bm25 = Bm25("Qdrant/bm25") + + qdrant = AsyncQdrantClient(url=url) + fallback = limit + 1 + + async with httpx.AsyncClient() as http: + results: list[Result] = [] + + for i, (query, expected, kind) in enumerate(cases, 1): + print(f" [{i}/{len(cases)}] {query!r}…", file=sys.stderr, end="\r") + dense = await get_dense_vector(http, embeddings_url, query) + sparse = await get_sparse_vector(bm25, query) + + hybrid_names = await search_hybrid(qdrant, collection, dense, sparse, limit) + dense_names = await search_dense(qdrant, collection, dense, limit) + + results.append( + Result( + query=query, + expected=expected, + kind=kind, + dense_rank=get_rank(dense_names, expected, fallback), + hybrid_rank=get_rank(hybrid_names, expected, fallback), + ) + ) + + print(" " * 60, file=sys.stderr, end="\r") + print(f"Results for {len(results)} queries (search limit={limit})\n", flush=True) + passed = print_report(results, limit) + + await qdrant.close() + sys.exit(0 if passed else 1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--url", + default=os.getenv("QDRANT_URL", "http://localhost:6333"), + help="Qdrant URL (default: $QDRANT_URL or http://localhost:6333)", + ) + parser.add_argument( + "--collection", + default=os.getenv("QDRANT_COLLECTION", "code_symbols"), + help="Collection name (default: $QDRANT_COLLECTION or code_symbols)", + ) + parser.add_argument( + "--embeddings-url", + default=os.getenv("EMBEDDINGS_URL", "http://localhost:8087"), + help="Jina TEI server URL (default: $EMBEDDINGS_URL or http://localhost:8087)", + ) + parser.add_argument( + "--limit", + type=int, + default=DEFAULT_LIMIT, + metavar="N", + help=f"Number of results to fetch per search (default: {DEFAULT_LIMIT})", + ) + parser.add_argument( + "--test-cases-file", + default=None, + metavar="FILE", + help="JSON file produced by seed_test_cases.py --output-json. " + "Overrides the hardcoded TEST_CASES list.", + ) + parser.add_argument( + "--kinds", + nargs="+", + choices=["exact", "tokenized", "snake_case", "prefix", "semantic"], + default=None, + metavar="KIND", + help="Only validate test cases of these kinds (default: all).", + ) + args = parser.parse_args() + asyncio.run(main( + args.url, args.collection, args.embeddings_url, args.limit, + args.test_cases_file, + set(args.kinds) if args.kinds else None, + ))