diff --git a/.claude/skills/codemap/SKILL.md b/.claude/skills/codemap/SKILL.md index 441ffe59..26eda6b7 100644 --- a/.claude/skills/codemap/SKILL.md +++ b/.claude/skills/codemap/SKILL.md @@ -17,6 +17,7 @@ codemap --deps # Dependency flow (imports/functions/hubs) codemap --diff # Changes vs main branch codemap --diff --ref # Changes vs specific branch codemap --importers # Who imports this file? Is it a hub? +codemap find "" # Where is X defined (names first, then body text); grep exact symptoms codemap handoff . # Build + save handoff artifact codemap handoff --latest . # Read latest saved handoff codemap handoff --json . # Machine-readable handoff payload diff --git a/.claude/skills/explore/SKILL.md b/.claude/skills/explore/SKILL.md index 63b4c120..d28fe0e6 100644 --- a/.claude/skills/explore/SKILL.md +++ b/.claude/skills/explore/SKILL.md @@ -28,7 +28,8 @@ description: Systematic codebase exploration using codemap's analysis tools. Use | Question | Command | |----------|---------| -| Where is X defined? | `codemap --deps .` → find the file | +| Where is X defined? | `codemap find ""` | +| Where is this bug? | `codemap find ""` scores names first, then body text (`(body)` hits); grep for the exact symptom text | | Who uses X? | `codemap --importers ` | | What's the architecture? | `codemap --deps .` → look at hub files | | What changed recently? | `codemap --diff` | diff --git a/README.md b/README.md index 97d42d79..64dbb295 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ codemap Coverage: complete ``` -**Where is the code that does X.** `codemap find "hub importers"` +**Where is X defined.** `codemap find "hub importers"` ranks files by path and function/type names, then by body text; hits that matched only in a body are tagged `(body)`. For an exact bug symptom, grep too. ``` main.go @@ -214,7 +214,7 @@ codemap --deps . # dependency flow codemap --importers f # who imports a file codemap blast-radius # review bundle: diff + deps + importers codemap collide # rank open PRs by shared-file merge-order hazard -codemap find "" # rank files by path and symbol match, with importer counts +codemap find "" # where is X defined: rank by path, symbol names, then body text ("(body)" tag), with importer counts codemap handoff . # save layered handoff for cross-agent continuation codemap context # machine-readable project context JSON codemap doctor # validate agent integrations diff --git a/find/find.go b/find/find.go index 8f3316d3..33dcff78 100644 --- a/find/find.go +++ b/find/find.go @@ -1,7 +1,9 @@ // Package find ranks project files against a natural-language query using -// only what the dependency scanner already extracts: file paths and the -// function names inside them. It is lexical (BM25 over split identifiers), -// not semantic; a query has to share vocabulary with the code it wants. +// what the dependency scanner already extracts (file paths and the +// function/type names inside them) plus the words in each file's body. It is +// lexical (BM25 over split identifiers), not semantic; a query has to share +// vocabulary with the code it wants. Hits that matched only in a body are +// tagged "(body)". package find import ( @@ -9,6 +11,8 @@ import ( "fmt" "io" "math" + "os" + "path/filepath" "sort" "strings" "unicode" @@ -26,7 +30,9 @@ const ( // exactBonus is added once per query word that is a whole word of a // symbol or the file basename, so "theme" pulls loadTheme ahead of a file // whose path merely shares a directory token. - exactBonus = 1.0 + exactBonus = 1.0 + bodyTag = "(body)" + maxBodySize = 1 << 20 ) type Hit struct { @@ -51,7 +57,12 @@ func Run(ctx context.Context, root string, filters scanner.Filters, query string if err != nil { return Report{}, err } - report := Report{Schema: Schema, Query: query, Hits: Rank(outcome.Analyses, query, limit)} + body, err := bodyHits(ctx, root, outcome.Analyses, query) + if err != nil { + return Report{}, err + } + hits := mergeBody(rankAll(outcome.Analyses, query), body) + report := Report{Schema: Schema, Query: query, Hits: truncate(hits, limit)} sources := outcome.Sources if fg, graphErr := scanner.BuildFileGraphFromOutcome(ctx, root, outcome, filters); graphErr == nil { sources = fg.Coverage.Sources @@ -68,9 +79,10 @@ func Run(ctx context.Context, root string, filters scanner.Filters, query string // Rank scores every analysis against query and returns the top limit hits // with a positive score. Importer fields are left zero; Run fills them. func Rank(analyses []scanner.FileAnalysis, query string, limit int) []Hit { - if limit <= 0 { - limit = DefaultLimit - } + return truncate(rankAll(analyses, query), limit) +} + +func rankAll(analyses []scanner.FileAnalysis, query string) []Hit { queryTokens := tokenize(query) if len(queryTokens) == 0 { return nil @@ -115,15 +127,98 @@ func Rank(analyses []scanner.FileAnalysis, query string, limit int) []Hit { } hits = append(hits, Hit{Path: a.Path, Score: round(score), Matched: matched}) } + sortHits(hits) + return hits +} + +func sortHits(hits []Hit) { sort.SliceStable(hits, func(i, j int) bool { if hits[i].Score != hits[j].Score { return hits[i].Score > hits[j].Score } return hits[i].Path < hits[j].Path }) +} + +func truncate(hits []Hit, limit int) []Hit { + if limit <= 0 { + limit = DefaultLimit + } if len(hits) > limit { - hits = hits[:limit] + return hits[:limit] + } + return hits +} + +// bodyHits scores the indexed source files by which query tokens their +// contents contain, each token weighted by its rarity across those files. +// A file needs two distinct query tokens (one for a one-word query). +// ponytail: reads every indexed file under 1 MB on each query; add a +// content index if find on large repos gets slow. +func bodyHits(ctx context.Context, root string, analyses []scanner.FileAnalysis, query string) ([]Hit, error) { + queryTokens := unique(tokenize(query)) + need := min(2, len(queryTokens)) + if need == 0 { + return nil, ctx.Err() + } + found := make([][]string, len(analyses)) + df := make(map[string]int) + for i, a := range analyses { + if err := ctx.Err(); err != nil { + return nil, err + } + path := a.Path + if !filepath.IsAbs(path) { + path = filepath.Join(root, path) + } + info, err := os.Stat(path) + if err != nil || info.Size() > maxBodySize { + continue + } + data, err := os.ReadFile(path) + if err != nil { + continue + } + words := make(map[string]struct{}) + for _, w := range tokenize(string(data)) { + words[w] = struct{}{} + } + for _, q := range queryTokens { + if _, ok := words[q]; ok { + found[i] = append(found[i], q) + df[q]++ + } + } + } + n := float64(len(analyses)) + var hits []Hit + for i, a := range analyses { + if len(found[i]) < need { + continue + } + score := 0.0 + for _, q := range found[i] { + score += math.Log((n-float64(df[q])+0.5)/(float64(df[q])+0.5) + 1) + } + hits = append(hits, Hit{Path: a.Path, Score: round(score), Matched: []string{bodyTag}}) + } + return hits, nil +} + +// mergeBody adds body scores to name hits and appends body-only files. +func mergeBody(hits, body []Hit) []Hit { + at := make(map[string]int, len(hits)) + for i, h := range hits { + at[h.Path] = i + } + for _, b := range body { + if i, ok := at[b.Path]; ok { + hits[i].Score = round(hits[i].Score + b.Score) + continue + } + hits = append(hits, b) } + sortHits(hits) return hits } diff --git a/find/find_test.go b/find/find_test.go index 568979a8..1dd43e1e 100644 --- a/find/find_test.go +++ b/find/find_test.go @@ -2,6 +2,9 @@ package find import ( "bytes" + "context" + "os" + "path/filepath" "strings" "testing" @@ -46,3 +49,62 @@ func TestRenderShape(t *testing.T) { t.Fatalf("render =\n%s\nwant\n%s", buf.String(), want) } } + +func TestRunFallsBackToBodyText(t *testing.T) { + root := t.TempDir() + files := map[string]string{ + "go.mod": "module fixture\n\ngo 1.24\n", + "routing/matcher.go": "package routing\n\n// Each converter turns a segment into values.\nfunc Match(rule string) []string {\n\tvar values []string\n\tfor _, converter := range rule {\n\t\tvalues = append(values, string(converter))\n\t}\n\treturn values\n}\n", + "routing/rules.go": "package routing\n\nfunc Compile() {}\n", + "wrappers/request.go": "package wrappers\n\nfunc Parse() {}\n", + "wrappers/notes.go": "package wrappers\n\n// match converter values\nfunc Notes() {}\n", + } + for name, body := range files { + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + report, err := Run(context.Background(), root, scanner.Filters{}, "wrong converter values segment order", 5) + if err != nil { + t.Fatal(err) + } + if len(report.Hits) == 0 || report.Hits[0].Path != "routing/matcher.go" { + t.Fatalf("body-only match should rank first, got %+v", report.Hits) + } + if got := strings.Join(report.Hits[0].Matched, ","); got != "(body)" { + t.Fatalf("matched = %q, want (body)", got) + } + + // Same body words, but matcher.go also matches by name: it outranks the + // body-only file and keeps its name tag. + report, err = Run(context.Background(), root, scanner.Filters{}, "match converter values", 5) + if err != nil { + t.Fatal(err) + } + if len(report.Hits) < 2 || report.Hits[0].Path != "routing/matcher.go" || report.Hits[1].Path != "wrappers/notes.go" { + t.Fatalf("name+body should outrank body-only, got %+v", report.Hits) + } + if got := strings.Join(report.Hits[0].Matched, ","); got != "Match" { + t.Fatalf("matched = %q, want Match", got) + } +} + +func TestBodyHitsHonorsCancellation(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "a.go"), []byte("package a\n// converter values\n"), 0o644); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + hits, err := bodyHits(ctx, root, []scanner.FileAnalysis{{Path: "a.go"}}, "converter values") + if err == nil || len(hits) != 0 { + t.Fatalf("cancelled ctx: hits=%+v err=%v, want no hits and an error", hits, err) + } + if _, err := Run(ctx, root, scanner.Filters{}, "converter values", 5); err == nil { + t.Fatal("Run with cancelled ctx returned no error") + } +} diff --git a/find_cmd.go b/find_cmd.go index 97653ac8..1fd1192c 100644 --- a/find_cmd.go +++ b/find_cmd.go @@ -24,9 +24,10 @@ func runFindSubcommand(args []string, launchDir string) int { fs.Usage = func() { fmt.Fprintln(os.Stderr, "Usage: codemap find [options] ") fmt.Fprintln(os.Stderr) - fmt.Fprintln(os.Stderr, "Rank files by how well their path and function names match the query,") + fmt.Fprintln(os.Stderr, "Rank files by how well their path and function/type names match the query,") fmt.Fprintln(os.Stderr, "annotated with importer counts so a hit also says how risky it is to edit.") - fmt.Fprintln(os.Stderr, "Lexical only: the query must share words with the code it is looking for.") + fmt.Fprintln(os.Stderr, "Good for \"where is X defined\". Also scores file contents; hits that matched") + fmt.Fprintln(os.Stderr, "only in a body are tagged \"(body)\". For a free-text symptom, grep as well.") fmt.Fprintln(os.Stderr) fmt.Fprintln(os.Stderr, "Options:") fs.PrintDefaults() diff --git a/main.go b/main.go index d8b8aee9..9ceb4810 100644 --- a/main.go +++ b/main.go @@ -347,7 +347,7 @@ func main() { fmt.Println(" codemap handoff [path] # Build handoff artifact for agent switching") fmt.Println(" codemap blast-radius [path] # Compact bounded blast-radius bundle") fmt.Println(" codemap collide # Rank open PRs by shared-file merge-order hazard") - fmt.Println(" codemap find # Rank files by path and symbol match, with importer counts") + fmt.Println(" codemap find # Where is X defined: rank by path, symbol names, then body text") fmt.Println() fmt.Println("Project config:") fmt.Println(" codemap config init # Create .codemap/config.json (auto-detects extensions)") diff --git a/mcp/main.go b/mcp/main.go index 452ce56a..3eef2699 100644 --- a/mcp/main.go +++ b/mcp/main.go @@ -274,7 +274,7 @@ func NewServer(options RuntimeOptions) *mcp.Server { // Tool: find - Rank files against a query using paths and function names mcp.AddTool(server, &mcp.Tool{ Name: "find", - Description: "Rank project files by how well their path and function names match a query (lexical BM25 over split identifiers). Each hit reports importer count and hub status, so it says where the code is and how risky it is to edit. Use before grep when you know what the code does but not where it lives.", + Description: "Rank project files by how well their path and function/type names match a query (lexical BM25 over split identifiers). Good for \"where is X defined\" when the query uses words from the code's names. Each hit reports importer count and hub status, so it also says how risky the file is to edit. Also scores file contents; hits that matched only in a body are tagged \"(body)\". For a free-text symptom, grep the body text too.", }, handleFind) // Tool: status - Verify MCP connection diff --git a/plugins/codemap/skills/codemap/SKILL.md b/plugins/codemap/skills/codemap/SKILL.md index 441ffe59..26eda6b7 100644 --- a/plugins/codemap/skills/codemap/SKILL.md +++ b/plugins/codemap/skills/codemap/SKILL.md @@ -17,6 +17,7 @@ codemap --deps # Dependency flow (imports/functions/hubs) codemap --diff # Changes vs main branch codemap --diff --ref # Changes vs specific branch codemap --importers # Who imports this file? Is it a hub? +codemap find "" # Where is X defined (names first, then body text); grep exact symptoms codemap handoff . # Build + save handoff artifact codemap handoff --latest . # Read latest saved handoff codemap handoff --json . # Machine-readable handoff payload diff --git a/plugins/codemap/skills/explore/SKILL.md b/plugins/codemap/skills/explore/SKILL.md index 63b4c120..d28fe0e6 100644 --- a/plugins/codemap/skills/explore/SKILL.md +++ b/plugins/codemap/skills/explore/SKILL.md @@ -28,7 +28,8 @@ description: Systematic codebase exploration using codemap's analysis tools. Use | Question | Command | |----------|---------| -| Where is X defined? | `codemap --deps .` → find the file | +| Where is X defined? | `codemap find ""` | +| Where is this bug? | `codemap find ""` scores names first, then body text (`(body)` hits); grep for the exact symptom text | | Who uses X? | `codemap --importers ` | | What's the architecture? | `codemap --deps .` → look at hub files | | What changed recently? | `codemap --diff` | diff --git a/skills/builtin/codemap.md b/skills/builtin/codemap.md index 48cd2b32..ef03ac12 100644 --- a/skills/builtin/codemap.md +++ b/skills/builtin/codemap.md @@ -19,6 +19,7 @@ codemap --deps # Dependency flow (imports/functions/hubs) codemap --diff # Changes vs main branch codemap --diff --ref # Changes vs specific branch codemap --importers # Who imports this file? Is it a hub? +codemap find "" # Where is X defined (names first, then body text); grep exact symptoms codemap handoff . # Build + save handoff artifact codemap handoff --latest . # Read latest saved handoff codemap handoff --json . # Machine-readable handoff payload diff --git a/skills/builtin/explore.md b/skills/builtin/explore.md index dd65fdac..0c4f127e 100644 --- a/skills/builtin/explore.md +++ b/skills/builtin/explore.md @@ -31,7 +31,8 @@ languages: ["go", "typescript", "python", "java", "rust", "ruby", "swift"] | Question | Command | |----------|---------| -| Where is X defined? | `codemap --deps .` → find the file | +| Where is X defined? | `codemap find ""` | +| Where is this bug? | `codemap find ""` scores names first, then body text (`(body)` hits); grep for the exact symptom text | | Who uses X? | `codemap --importers ` | | What's the architecture? | `codemap --deps .` → look at hub files | | What changed recently? | `codemap --diff` |