From c7cd18e6d98172a25d75de6c06ced7c5bda2f243 Mon Sep 17 00:00:00 2001 From: lick <2188718831@qq.com> Date: Thu, 12 Nov 2026 20:05:36 -0800 Subject: [PATCH] feat: navigate into the jars old projects keep in their tree Tested on niche frameworks and old projects (JFinal, Nutz, Solon, Ant, Struts 1, JForum 2): every target lands on its declaration, but JForum 2, an Ant project with its jars committed to WEB-INF/lib and no pom, found none of its dependencies, and the second variable of a declaration of several (int i = 0, size = s.length()) was never found. - deps.TreeJars lists the jars a working tree keeps: the lib and M2_REPO entries of Eclipse .classpath files in their order (with sourcepath), then jars with a lib or libs directory up to three levels above them (WEB-INF/lib, lib/optional), probing the usual lib directories next to build.xml, .classpath and .project even when .gitignore leaves them out. Test resource jars, the Gradle wrapper and build output are not dependencies. A -sources.jar or -src.zip next to a jar is its source. - A source archive is used only when one of the jar's classes is at its place in it: a zip of a source tree would name every class wrongly, so the class jar's stubs are used instead. - Dependencies are now switched on by their own option rather than by a Maven repository being set, and a change to a .classpath file or to any tree jar rereads them. - findLocal gives the variables after the commas of a declaration the first one's type; names in the initializers' arguments are deeper in parentheses and stay unaffected. - nav-bench -under samples another source root (src/, src/main/org/). JForum 2 resolves 99.3% of 1,500 positions (was 95.9%); JFinal, Nutz and Ant gain 4-7 positions each; Nacos and ruoyi-vue-pro are unchanged, and an A/B run shows no change in latency or memory. Co-Authored-By: Claude Opus 5.5 --- README.md | 29 +++- cmd/java-navigation/main.go | 9 +- cmd/nav-bench/main.go | 10 +- docs/releases/v0.0.1.md | 4 +- internal/codenav/UPSTREAM.md | 2 + internal/codenav/resolve.go | 27 ++++ internal/codenav/usages_test.go | 49 +++++++ internal/deps/jars.go | 43 ++++-- internal/deps/tree.go | 226 ++++++++++++++++++++++++++++++++ internal/deps/tree_test.go | 121 +++++++++++++++++ internal/lsp/server.go | 8 +- internal/workspace/deps_test.go | 44 ++++++- internal/workspace/workspace.go | 91 ++++++++----- 13 files changed, 609 insertions(+), 54 deletions(-) create mode 100644 internal/deps/tree.go create mode 100644 internal/deps/tree_test.go diff --git a/README.md b/README.md index f79b339..d11dde1 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ | Lombok | `@Data`/`@Getter`/`@Setter` 生成的访问器跳到字段，`@Builder`、`@Slf4j`、构造器注解跳到注解本身（与 JDT LS 一致） | | 跳进 JDK | 项目里找不到的名字到本机 JDK 的 `src.zip` 里找，`String`、`List`、`hashCode()` 等都能跳，并能在 JDK 源码里继续跳 | | 跳进依赖库 | Maven 项目的依赖（含传递依赖）从本地仓库读：下载过 `-sources.jar` 的打开源码，否则打开从 `.class` 生成的签名存根（类、成员类、字段、构造器、方法，参数名取自调试信息），并能在里面继续跳；`@Schema(description = ...)`、`log.info(...)`、`StrUtil.isEmpty(...)` 都能跳 | +| 老项目的 jar | 没有 Maven 的 Ant、Eclipse、MyEclipse 项目，读 `.classpath` 里列出的 jar（含 `M2_REPO` 变量和 `sourcepath`），以及 `lib/`、`libs/`、`WEB-INF/lib` 里的 jar | | 生成代码 | 模块的 `target/generated-sources`（protobuf、注解处理器）和 Gradle 的 `build/generated` 也进索引，即使被 `.gitignore` 忽略 | | 查找引用 | 只列出能跳回这个声明的出现，所以列表和在任一行上跳转的结果一致；`package`/`import` 不算引用 | | 文档结构 | 类型、成员和嵌套类型的大纲（`textDocument/documentSymbol`） | @@ -37,7 +38,14 @@ 本地仓库没有的 jar 跳过；版本是范围或引用了不认识的属性时取本地最高的版本。工作区自己的模块不当依赖。 只预先列出每个 jar 的目录（Nacos 265 个 jar、4.4 万个类；ruoyi-vue-pro 824 个 jar、15 万个类，约 0.3 秒）， 某个类第一次被查到时才读它的源码或生成存根；跳进去时把那一个文件写成只读副本，放在 `${stateDir}/dependency-sources`， - 不写本地仓库。`pom.xml` 变了、或距上次读取超过 5 分钟，下一次扫描重新读取。 + 不写本地仓库。 +- 工作区里的 jar：先按每个 Eclipse `.classpath` 的 `lib` 条目和 `M2_REPO` 变量条目的顺序，再加上路径往上三层内 + 有 `lib` 或 `libs` 目录的 jar（`WEB-INF/lib`、`lib/optional` 都算；测试资源里的 jar、`gradle-wrapper.jar`、 + `target`/`build` 里的不算）；工作区根目录和每个 `build.xml`、`.classpath`、`.project` 所在目录下的 `lib`、 + `WEB-INF/lib`、`WebContent/WEB-INF/lib`、`WebRoot/WEB-INF/lib` 等即使被 `.gitignore` 忽略也会读。源码取 + `.classpath` 的 `sourcepath`，或旁边的 `-sources.jar`、`-src.zip`；源码包不是按包名放的（比如整个源码树打的 zip）就不用，改用存根。 + 这些 jar 只读，不会被修改。 +- `pom.xml`、`.classpath` 或上面任一个 jar 变了、或距上次读取超过 5 分钟，下一次扫描重新读取依赖。 - JDK：按 `JAVA_HOME` → 系统默认 JDK（macOS 的 `/usr/libexec/java_home`，其他系统 `PATH` 上的 `java`）→ 常见安装目录 （含 IntelliJ 的 `~/.jdks`、SDKMAN!）的顺序找第一个带 `src.zip` 的 JDK。只预先列出 zip 目录，某个类第一次被查到时才解析。 跳进 JDK 时把那一个源码文件解压成只读副本，放在 Lithe 为插件分配的状态目录（`${stateDir}`）里，卸载插件时一起删除； @@ -76,9 +84,26 @@ 其余是 JDT LS 能跳、本插件跳不了的泛型和 lambda（`list.get(i).split(...)`、`map.get(a).get(b)`、比较器里的 `o1.getPriority()`），以及两处要按泛型实参挑选的重载（`JsonUtils.toObj(String, ...)` 和 `toObj(byte[], ...)`）。 +### 小众框架和老项目 + +每个项目 1500 个随机位置（种子 2），源码目录按项目实际布局取。跳到的目标全部逐个核对过，都落在该名字的声明行上： + +| 项目 | 类型 | 能跳转的比例 | 跳转 p50 / p99 | 第一次跳转 | 内存（物理占用） | +| --- | --- | --- | --- | --- | --- | +| [JFinal](https://github.com/jfinal/jfinal) | 国产小众 Web 框架，Maven | 96.8% | 0.13 ms / 1.5 ms | 57 ms | 21 MiB | +| [Nutz](https://github.com/nutzam/nutz) | 国产小众框架，源码直接放 `src/` | 97.8% | 0.15 ms / 1.3 ms | 55 ms | 27 MiB | +| [Solon](https://github.com/opensolon/solon) | 国产小众框架，150 个模块 | 96.5% | 0.12 ms / 2.1 ms | 0.19 s | 36 MiB | +| [Apache Ant](https://github.com/apache/ant) | 2000 年起的老项目，`src/main/org` | 99.1% | 0.19 ms / 1.3 ms | 56 ms | 29 MiB | +| [Struts 1](https://github.com/apache/struts1) | 已归档的老框架，JSF/JSP | 97.5% | 0.16 ms / 1.8 ms | 44 ms | 22 MiB | +| [JForum 2](https://github.com/rafaelsteil/jforum2) | 已归档的 Ant 项目，jar 提交在 `WEB-INF/lib`，没有 pom | 99.3%（读工作区 jar 之前 95.9%） | 0.15 ms / 1.4 ms | 37 ms | 20 MiB | + +剩下没跳到的主要是：依赖没在项目里声明或本地仓库没有（JFinal 的 Servlet 依赖在 pom 里被注释掉了，Struts 的 +JSF 依赖没下载过）、数组的 `.length`（JDT LS 也不跳）、`java.util.Date` 这种全限定名里的包名段，以及 lambda 参数。 + ## 限制 -- **依赖库只读 Maven 项目**：Gradle 项目的依赖还读不了（Gradle 的生成代码可以）；只用本地仓库里已有的 jar， +- **依赖库只读 Maven 和工作区里的 jar**：Gradle 项目的依赖还读不了（Gradle 的生成代码可以）；Ant 的 + `build.xml` 里写在 `lib` 之外目录的 classpath 也读不了。只用本地仓库里已有的 jar， 从没构建过、依赖还没下载的项目跳不进依赖库。没有 `-sources.jar` 时打开签名存根，没有方法体和注释； 编译时没带调试信息的接口和抽象方法，参数名是 `arg0`、`arg1`。 - **没有泛型推断**：重载先按参数个数选，个数相同时按实参里写得出来的类型挑（字面量、强转、`new` 表达式、 diff --git a/cmd/java-navigation/main.go b/cmd/java-navigation/main.go index 7964aee..c5df88b 100644 --- a/cmd/java-navigation/main.go +++ b/cmd/java-navigation/main.go @@ -35,7 +35,7 @@ func main() { state := flag.String("state-dir", "", "writable directory for extracted JDK sources (the host's ${stateDir})") printVersion := flag.Bool("version", false, "print the version and exit") noJDK := flag.Bool("no-jdk", false, "do not navigate into the JDK's sources") - noDeps := flag.Bool("no-dependencies", false, "do not navigate into the jars of Maven dependencies") + noDeps := flag.Bool("no-dependencies", false, "do not navigate into dependency jars (Maven dependencies and the jars the project keeps)") flag.Bool("stdio", true, "speak LSP over standard input and output (the only transport)") flag.Parse() if *printVersion { @@ -49,8 +49,11 @@ func main() { if !*noJDK && *state != "" { opt.FindJDK = jdk.FindJDK } - if home, err := os.UserHomeDir(); err == nil && !*noDeps && *state != "" { - opt.MavenRepo = deps.LocalRepository(home) + if !*noDeps && *state != "" { + opt.Dependencies = true + if home, err := os.UserHomeDir(); err == nil { + opt.MavenRepo = deps.LocalRepository(home) + } } if *state == "" { logger.Printf("no --state-dir: navigation into the JDK and dependency jars is off") diff --git a/cmd/nav-bench/main.go b/cmd/nav-bench/main.go index ff4e3a5..6f55e88 100644 --- a/cmd/nav-bench/main.go +++ b/cmd/nav-bench/main.go @@ -1,5 +1,6 @@ // Command nav-bench measures a Java language server on a real project: it -// samples identifiers in the project's src/main/java files (not keywords, +// samples identifiers in the project's src/main/java files (-under picks +// another source root; not keywords, // literals, comments, package or import lines), asks the server for the // definition of each one, then for the references of some, and reports the // share that resolved per syntactic category, the latency of each request @@ -349,14 +350,14 @@ func utf16Col(line string, byteOff int) int { return len(utf16.Encode([]rune(line[:byteOff]))) } -func collect(root string) []sample { +func collect(root, under string) []sample { out, err := exec.Command("git", "-C", root, "ls-files", "--cached", "--others", "--exclude-standard", "*.java").Output() if err != nil { panic(err) } var all []sample for _, rel := range strings.Split(strings.TrimSpace(string(out)), "\n") { - if !strings.Contains(rel, "src/main/java/") { + if !strings.Contains(rel, under) { continue } path := filepath.Join(root, rel) @@ -400,13 +401,14 @@ func main() { tag := flag.String("tag", "run", "output tag") outDir := flag.String("out", ".", "output dir") refs := flag.Int("refs", 20, "references requests") + under := flag.String("under", "src/main/java/", "sample only files whose path contains this (old layouts: src/)") flag.Parse() args := flag.Args() if *root == "" || len(args) == 0 { fmt.Fprintln(os.Stderr, "usage: nav-bench -root [-seed n] [-n samples] [-jdtls] [-tag name] [-out dir] -- ") os.Exit(2) } - all := collect(*root) + all := collect(*root, *under) r := rand.New(rand.NewSource(*seed)) r.Shuffle(len(all), func(i, j int) { all[i], all[j] = all[j], all[i] }) if len(all) > *n { diff --git a/docs/releases/v0.0.1.md b/docs/releases/v0.0.1.md index e6ff80c..8f246cb 100644 --- a/docs/releases/v0.0.1.md +++ b/docs/releases/v0.0.1.md @@ -6,12 +6,14 @@ - 跳进本机 JDK 的 `src.zip`，并能在 JDK 源码里继续跳；JDK 源码以只读副本打开，放在插件状态目录里。 - 跳进 Maven 依赖库：从工作区的 `pom.xml`、父 POM、BOM 和本地仓库的 POM 按 Maven 的规则找出依赖 jar（不运行 Maven、不联网）， 有 `-sources.jar` 打开源码，没有就打开从 `.class` 生成的签名存根；每个类第一次用到时才读取，副本同样只读、放在插件状态目录里。 +- 没有 Maven 的老项目（Ant、Eclipse、MyEclipse）：读 `.classpath` 列出的 jar 和 `lib/`、`WEB-INF/lib` 里的 jar，jar 只读、不修改。 +- 一条声明里的多个变量（`int i = 0, size = s.length();`）都能跳。 - 生成代码（`target/generated-sources`、Gradle 的 `build/generated`）也进索引。 - 字符串字面量、类字面量、`Iface.super` 和 Lombok `log` 字段后面的调用能继续跳。 - 查找引用：只列出能跳回该声明的出现。 - 文档结构（大纲）。 - 在 Nacos 上和 JDT LS 对比 4500 个位置，4481 个跳到同一处，能跳转的比例 98.2%–99.3%（JDT LS 98.5%–99.5%）； - ruoyi-vue-pro 98.4%–98.5%。跳转 p50 约 0.1 ms，常驻内存 60–90 MiB（JDT LS 约 3 GB）。 + ruoyi-vue-pro 98.4%–98.5%；JFinal、Nutz、Solon、Ant、Struts 1、JForum 2 为 96.5%–99.3%，目标全部核对无误。跳转 p50 约 0.1 ms，常驻内存 60–90 MiB（JDT LS 约 3 GB）。 - 打开的文件按编辑器里未保存的内容索引。 平台：macOS（Apple Silicon、Intel）、Windows（x64、ARM64），每个平台一个 ZIP。 diff --git a/internal/codenav/UPSTREAM.md b/internal/codenav/UPSTREAM.md index 6012d38..bf7742d 100644 --- a/internal/codenav/UPSTREAM.md +++ b/internal/codenav/UPSTREAM.md @@ -35,6 +35,8 @@ （`Foo.class.getName()`）和 `Iface.super.m()`；`parse.go` 记下 Lombok 日志注解生成的 logger 类型 （`Type.LogType`，`@Slf4j` 是 `org.slf4j.Logger`），`findField` 给 `log` 字段带上这个类型， `log.info(...)` 才能跳进日志库。 + - 局部变量：`resolve.go` 的 `findLocal` 认出一条声明里逗号后面的变量（`int i = 0, size = s.length();`、 + `int a, b;`），沿用第一个变量的类型；初始化表达式里括号更深的名字不算。 - 同步后运行 `go test ./...`。 `internal/jdk/find.go` 改编自 Viff 的 `internal/platform/jdk.go`（去掉了 Viff 的进程辅助函数， diff --git a/internal/codenav/resolve.go b/internal/codenav/resolve.go index 13d412d..3afd2ce 100644 --- a/internal/codenav/resolve.go +++ b/internal/codenav/resolve.go @@ -1095,16 +1095,38 @@ func (q *qctx) findLocal(name string, at int) (local, bool) { decls = append(decls, local{k, ts, te}) names = append(names, p.txt(k)) } + // A declaration of several variables (int i = 0, n = s.length();) + // gives the names after its commas the first one's type (Lithe + // addition). multi is that declaration while its statement lasts, at + // paren depth multiDepth; the initializers' arguments are deeper. + var multi *local + depth, multiDepth := 0, 0 for k := b.start; k <= at && k < len(p.toks); k++ { switch { case p.is(k, "{"): marks = append(marks, len(decls)) + multi = nil continue case p.is(k, "}"): if n := len(marks); n > 0 { decls, names = decls[:marks[n-1]], names[:marks[n-1]] marks = marks[:n-1] } + multi = nil + continue + case p.is(k, "("): + depth++ + continue + case p.is(k, ")"): + depth-- + if multi != nil && depth < multiDepth { + multi = nil + } + continue + case p.is(k, ";"): + if multi != nil && depth <= multiDepth { + multi = nil + } continue case p.is(k, "->"): if p.ident(k-1) && !p.is(k-2, ".") { @@ -1138,6 +1160,11 @@ func (q *qctx) findLocal(name string, at int) (local, bool) { } if ts := q.typeStartBack(te - 1); ts >= 0 { add(k, ts, te) + if !p.is(nx, ")") && !p.is(nx, ":") { + multi, multiDepth = &local{k, ts, te}, depth + } + } else if multi != nil && depth == multiDepth && p.is(k-1, ",") && !p.is(nx, ":") { + add(k, multi.typeS, multi.typ) } } for i := len(decls) - 1; i >= 0; i-- { diff --git a/internal/codenav/usages_test.go b/internal/codenav/usages_test.go index 3c408d7..2b6b39e 100644 --- a/internal/codenav/usages_test.go +++ b/internal/codenav/usages_test.go @@ -185,3 +185,52 @@ func contains(s []string, v string) bool { } return false } + +// Every variable of a declaration of several is a local with the +// declaration's type; names inside the initializers are not (Lithe +// addition). +func TestEveryVariableOfADeclarationIsALocal(t *testing.T) { + const path = "src/a/M.java" + src := `package a; + +class M { + int b; + String name() { return ""; } + M make(int k, int b) { return this; } + int run(String key, java.util.List list) { + int s2, s5; + for (int i = 0, size = key.length(); i < size; i++) { + s5 = i; + } + M x = list.get(0), y = make(s2, b), z; + z = y; + return s5 + y.name().length() + b; + } +} +` + ix := NewIndex() + ix.Put(ParseFile(path, []byte(src))) + line := func(needle string) int { + t.Helper() + return strings.Count(src[:strings.Index(src, needle)], "\n") + } + for _, c := range []struct { + needle string + skip int + wantLocal bool + wantLine int + }{ + {"s5 = i", 0, true, line("int s2, s5;")}, + {"size; i++", 0, true, line("for (int i")}, + {"z = y", 0, true, line("M x =")}, + {"y;\n", 0, true, line("M x =")}, + {"name().length", 0, false, line("String name()")}, + // b in make(s2, b) is the field, not a third variable of M x. + {"b;\n }\n}", 0, false, line("int b;")}, + } { + l, ok := ix.Resolve(path, []byte(src), offsetOf(t, src, c.needle, 0, c.skip)) + if !ok || l.Local != c.wantLocal || l.Line != c.wantLine { + t.Errorf("%q: %+v ok=%v, want local=%v line %d", c.needle, l, ok, c.wantLocal, c.wantLine) + } + } +} diff --git a/internal/deps/jars.go b/internal/deps/jars.go index 9f956cb..144426d 100644 --- a/internal/deps/jars.go +++ b/internal/deps/jars.go @@ -1,6 +1,7 @@ // Package deps lets navigation follow names into a project's dependency // jars. It finds the jars of the Maven projects in a working tree in the -// local repository (MavenJars), lists the classes each one declares and, +// local repository (MavenJars) and the jars a tree keeps itself (TreeJars), +// lists the classes each one declares and, // for a class a lookup needs, returns Java source: the entry of the jar's // -sources.jar when it was downloaded, else a stub of signatures written // from the class files. Only the archives' directories are read up front; @@ -127,18 +128,42 @@ func (s *Set) Paths() []string { return s.paths } // as the .java file a stub of them is. func listJar(j Jar) (*archive, []string, error) { a := &archive{Jar: j} + a.file = j.Path + classes, err := a.list() + if err != nil { + return nil, nil, err + } if j.Sources != "" { - a.file, a.sources = j.Sources, true - if entries, err := a.list(); err == nil && len(entries) > 0 { - return a, entries, nil + s := &archive{Jar: j, file: j.Sources, sources: true} + if entries, err := s.list(); err == nil && rootedLike(entries, classes) { + return s, entries, nil } } - a.file, a.sources = j.Path, false - entries, err := a.list() - if err != nil { - return nil, nil, err + return a, classes, nil +} + +// rootedLike reports whether a source archive lays its files out by package +// from its root, as the class jar does: an archive of a source tree +// ("lib-1.0-src/src/main/java/a/B.java") would name every class wrongly. +// One class of the jar found at its place is enough, since a sources jar may +// leave out generated classes. A jar without classes accepts any sources. +func rootedLike(sources, classes []string) bool { + if len(sources) == 0 { + return false + } + if len(classes) == 0 { + return true + } + have := make(map[string]bool, len(sources)) + for _, e := range sources { + have[e] = true + } + for _, c := range classes { + if have[c] { + return true + } } - return a, entries, nil + return false } func (a *archive) list() ([]string, error) { diff --git a/internal/deps/tree.go b/internal/deps/tree.go new file mode 100644 index 0000000..a9ce114 --- /dev/null +++ b/internal/deps/tree.go @@ -0,0 +1,226 @@ +package deps + +import ( + "encoding/xml" + "os" + "path" + "path/filepath" + "strings" +) + +// Projects older than Maven, and many built with Ant or exported from +// Eclipse or MyEclipse, keep their jars in the working tree instead of +// resolving them from a repository: WEB-INF/lib of a web application, a lib +// directory the build.xml adds to the classpath, the library entries of an +// Eclipse .classpath file. TreeJars finds those jars. + +// maxTreeJars bounds the jars one working tree contributes. +const maxTreeJars = 2000 + +// libDirs are the names of directories whose jars are on a build's +// classpath. A jar counts when one of the three directories above it has +// such a name, so lib/optional/junit.jar and WEB-INF/lib/x.jar do, and a +// test's resources jar or gradle/wrapper/gradle-wrapper.jar do not. +var libDirs = map[string]bool{"lib": true, "libs": true} + +// skippedTreeDirs hold build output and copies of jars found elsewhere. +var skippedTreeDirs = map[string]bool{"target": true, "build": true, "out": true, "bin": true, "node_modules": true, ".git": true} + +// libProbes are directories, relative to the tree's root and to each +// directory holding a build.xml, .classpath or .project, whose jars are read +// even when .gitignore leaves them out of the listing (Ant and Ivy often +// download them there). +var libProbes = []string{"lib", "libs", "WEB-INF/lib", "WebContent/WEB-INF/lib", "WebRoot/WEB-INF/lib", + "web/WEB-INF/lib", "src/main/webapp/WEB-INF/lib"} + +// projectMarkers name the files whose directory is probed for libProbes. +var projectMarkers = map[string]bool{"build.xml": true, ".classpath": true, ".project": true} + +// TreeJars lists the jars root keeps in its working tree: the library +// entries of its Eclipse .classpath files, in their order, then the jars in +// lib directories (see libDirs and libProbes). files are root's files as +// slash-separated relative paths; repo resolves Eclipse's M2_REPO variable +// and may be "". Jars that do not exist are left out. A jar's sources are the +// .classpath entry's sourcepath, else a -sources.jar or -src.zip next to it. +func TreeJars(root string, files []string, repo string) []Jar { + t := treeJars{seen: map[string]bool{}} + var markerDirs []string + listed := map[string]bool{} + for _, f := range files { + listed[f] = true + dir, base := path.Split(f) + if base == ".classpath" { + t.classpath(root, f, repo) + } + if projectMarkers[base] { + markerDirs = append(markerDirs, strings.TrimSuffix(dir, "/")) + } + } + for _, f := range files { + if inLibDir(f) { + t.add(filepath.Join(root, filepath.FromSlash(f)), "") + } + } + probed := map[string]bool{} + for _, d := range append([]string{""}, markerDirs...) { + for _, p := range libProbes { + dir := path.Join(d, p) + if !probed[dir] { + probed[dir] = true + t.walkLib(filepath.Join(root, filepath.FromSlash(dir)), 0) + } + } + } + return t.out +} + +type treeJars struct { + out []Jar + seen map[string]bool +} + +// add lists jar once, with sources when given or found next to it. +func (t *treeJars) add(jar, sources string) { + jar = filepath.Clean(jar) + if len(t.out) >= maxTreeJars || t.seen[jar] || !isLibraryJar(filepath.Base(jar)) { + return + } + if fi, err := os.Stat(jar); err != nil || !fi.Mode().IsRegular() { + return + } + t.seen[jar] = true + if sources == "" { + sources = sourcesNextTo(jar) + } else if fi, err := os.Stat(sources); err != nil || !fi.Mode().IsRegular() { + sources = "" + } + t.out = append(t.out, Jar{Coord: strings.TrimSuffix(filepath.Base(jar), ".jar"), Path: jar, Sources: sources}) +} + +// walkLib adds the jars under dir, two directory levels deep. +func (t *treeJars) walkLib(dir string, depth int) { + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, e := range entries { + p := filepath.Join(dir, e.Name()) + switch { + case e.IsDir() && depth < 2 && !skippedTreeDirs[e.Name()] && !strings.HasPrefix(e.Name(), "."): + t.walkLib(p, depth+1) + case e.Type().IsRegular() && strings.HasSuffix(e.Name(), ".jar"): + t.add(p, "") + } + } +} + +// classpathFile is the part of an Eclipse .classpath that names jars. +type classpathFile struct { + Entries []struct { + Kind string `xml:"kind,attr"` + Path string `xml:"path,attr"` + SourcePath string `xml:"sourcepath,attr"` + } `xml:"classpathentry"` +} + +// classpath adds the lib and M2_REPO var entries of the .classpath file rel. +func (t *treeJars) classpath(root, rel, repo string) { + b, err := readLimited(filepath.Join(root, filepath.FromSlash(rel)), maxPOMSize) + if err != nil { + return + } + var cp classpathFile + dec := xml.NewDecoder(strings.NewReader(string(b))) + dec.CharsetReader = charsetReader + if dec.Decode(&cp) != nil { + return + } + project := filepath.Join(root, filepath.FromSlash(path.Dir(rel))) + resolve := func(kind, p string) string { + if p == "" { + return "" + } + switch kind { + case "var": + v, rest, _ := strings.Cut(filepath.ToSlash(p), "/") + if v != "M2_REPO" || repo == "" || rest == "" { + return "" + } + return filepath.Join(repo, filepath.FromSlash(rest)) + case "lib": + if filepath.IsAbs(p) { + if _, err := os.Stat(p); err == nil { + return p + } + // "/Other/lib/x.jar" names a jar of the Eclipse project Other, + // usually a sibling folder. + return filepath.Join(filepath.Dir(project), filepath.FromSlash(strings.TrimPrefix(filepath.ToSlash(p), "/"))) + } + return filepath.Join(project, filepath.FromSlash(p)) + } + return "" + } + for _, e := range cp.Entries { + if jar := resolve(e.Kind, e.Path); jar != "" { + t.add(jar, resolve(e.Kind, e.SourcePath)) + } + } +} + +// inLibDir reports whether the listed file rel is a jar in a lib directory +// and not under build output. +func inLibDir(rel string) bool { + if !strings.HasSuffix(rel, ".jar") { + return false + } + dirs := strings.Split(path.Dir(rel), "/") + for _, d := range dirs { + if skippedTreeDirs[d] { + return false + } + } + for i := len(dirs) - 1; i >= 0 && i >= len(dirs)-3; i-- { + if libDirs[dirs[i]] { + return true + } + } + return false +} + +// isLibraryJar leaves out the source and documentation jars that sit next +// to the jars they describe. +func isLibraryJar(name string) bool { + stem, ok := strings.CutSuffix(name, ".jar") + if !ok { + return false + } + for _, s := range []string{"-sources", "-src", "-javadoc"} { + if strings.HasSuffix(stem, s) { + return false + } + } + return true +} + +// readLimited reads the regular file p when it is at most limit bytes. +func readLimited(p string, limit int64) ([]byte, error) { + fi, err := os.Stat(p) + if err != nil { + return nil, err + } + if !fi.Mode().IsRegular() || fi.Size() > limit { + return nil, os.ErrInvalid + } + return os.ReadFile(p) +} + +// sourcesNextTo finds the source archive a tree keeps next to jar. +func sourcesNextTo(jar string) string { + stem := strings.TrimSuffix(jar, ".jar") + for _, s := range []string{"-sources.jar", "-src.jar", "-sources.zip", "-src.zip"} { + if fi, err := os.Stat(stem + s); err == nil && fi.Mode().IsRegular() { + return stem + s + } + } + return "" +} diff --git a/internal/deps/tree_test.go b/internal/deps/tree_test.go new file mode 100644 index 0000000..9d1822a --- /dev/null +++ b/internal/deps/tree_test.go @@ -0,0 +1,121 @@ +package deps + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// treeJar writes an empty jar at rel under root and returns rel. +func treeJar(t *testing.T, root, rel string) string { + t.Helper() + p := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + writeJar(t, p, map[string]string{"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n"}) + return rel +} + +func relJars(t *testing.T, root string, jars []Jar) string { + t.Helper() + var out []string + for _, j := range jars { + rel, err := filepath.Rel(root, j.Path) + if err != nil { + t.Fatal(err) + } + s := filepath.ToSlash(rel) + if j.Sources != "" { + src, _ := filepath.Rel(root, j.Sources) + s += "+" + filepath.ToSlash(src) + } + out = append(out, s) + } + return strings.Join(out, " ") +} + +// A project without Maven keeps its jars in the tree: the .classpath entries +// come first, in their order, then the jars of lib directories, including +// those .gitignore leaves out of the listing; test resources, build output +// and the Gradle wrapper are not dependencies. +func TestTreeJarsOfAnAntAndEclipseProject(t *testing.T) { + root := t.TempDir() + repo := t.TempDir() + treeJar(t, repo, "org/x/m2lib/1.0/m2lib-1.0.jar") + files := []string{ + treeJar(t, root, "WEB-INF/lib/c3p0.jar"), + treeJar(t, root, "WEB-INF/lib/c3p0-sources.jar"), + treeJar(t, root, "lib/servlet-api.jar"), + treeJar(t, root, "lib/optional/junit-4.13.2.jar"), + treeJar(t, root, "tools/src/lib/zz.jar"), + treeJar(t, root, "src/etc/testcases/jars/signed.jar"), + treeJar(t, root, "gradle/wrapper/gradle-wrapper.jar"), + treeJar(t, root, "build/lib/app.jar"), + treeJar(t, root, "srcs/mail-src.jar"), + treeJar(t, root, "other/mail.jar"), + "build.xml", ".classpath", + } + // Not listed: Ivy downloads into lib/ and .gitignore leaves it out. + treeJar(t, root, "lib/ivy/commons-io.jar") + writeFile(t, filepath.Join(root, ".classpath"), ` + + + + + + + +`) + writeFile(t, filepath.Join(root, "build.xml"), "") + + got := TreeJars(root, files, repo) + rel := relJars(t, root, got) + m2 := filepath.ToSlash(filepath.Join(repo, "org/x/m2lib/1.0/m2lib-1.0.jar")) + rel = strings.Replace(rel, mustRel(t, root, m2), "M2_REPO/m2lib-1.0.jar", 1) + want := "other/mail.jar+srcs/mail-src.jar lib/servlet-api.jar M2_REPO/m2lib-1.0.jar " + + "WEB-INF/lib/c3p0.jar+WEB-INF/lib/c3p0-sources.jar lib/optional/junit-4.13.2.jar tools/src/lib/zz.jar " + + "lib/ivy/commons-io.jar" + if rel != want { + t.Errorf("TreeJars =\n %s\nwant\n %s", rel, want) + } + if got[0].Coord != "mail" { + t.Errorf("coordinate of other/mail.jar = %q", got[0].Coord) + } +} + +func mustRel(t *testing.T, root, p string) string { + t.Helper() + r, err := filepath.Rel(root, filepath.FromSlash(p)) + if err != nil { + t.Fatal(err) + } + return filepath.ToSlash(r) +} + +// A source archive of a source tree, not laid out by package, would name +// every class wrongly: the class jar's stubs are used instead. +func TestSourcesNotRootedAtThePackagesAreIgnored(t *testing.T) { + dir := t.TempDir() + lib, err := os.ReadFile(filepath.Join("testdata", "fixture", "lib-1.0.jar")) + if err != nil { + t.Fatal(err) + } + jar := writeFile(t, filepath.Join(dir, "lib.jar"), string(lib)) + srcTree := filepath.Join(dir, "lib-src.zip") + writeJar(t, srcTree, map[string]string{"lib-1.0/src/com/example/lib/Point.java": "package com.example.lib; public record Point(int x, int y) {}"}) + s := Open([]Jar{{Coord: "lib", Path: jar, Sources: srcTree}}, t.Logf) + p := pathOf(t, s, "com/example/lib/Point.java") + b, err := s.Read(p) + if err != nil || !strings.Contains(string(b), "Signatures read from lib.jar") { + t.Errorf("Point read %v:\n%s", err, b) + } + + rooted := filepath.Join(dir, "lib-sources.jar") + writeJar(t, rooted, map[string]string{"com/example/lib/Point.java": "package com.example.lib; public record Point(int x, int y) {}"}) + s = Open([]Jar{{Coord: "lib", Path: jar, Sources: rooted}}, t.Logf) + if b, err := s.Read(pathOf(t, s, "com/example/lib/Point.java")); err != nil || !strings.HasPrefix(string(b), "package com.example.lib;") { + t.Errorf("Point read from the sources %v:\n%s", err, b) + } +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 3688e73..685004f 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -27,8 +27,11 @@ type Options struct { State string // FindJDK locates the JDK whose sources are used; nil disables them. FindJDK func(context.Context) (jdk.JDK, bool) - // MavenRepo is the local Maven repository dependency jars are read - // from; "" disables them. + // Dependencies enables navigation into dependency jars (see + // workspace.Options.Dependencies). + Dependencies bool + // MavenRepo is the local Maven repository; "" leaves Maven dependencies + // out. MavenRepo string // Workspace overrides workspace options other than Root, for tests. Workspace workspace.Options @@ -308,6 +311,7 @@ func (s *Server) newWorkspace(root string) *workspace.Workspace { o.Root = root o.State = s.opt.State o.FindJDK = s.opt.FindJDK + o.Dependencies = s.opt.Dependencies o.MavenRepo = s.opt.MavenRepo if o.Logf == nil { o.Logf = s.opt.Logf diff --git a/internal/workspace/deps_test.go b/internal/workspace/deps_test.go index e5270ab..62c7485 100644 --- a/internal/workspace/deps_test.go +++ b/internal/workspace/deps_test.go @@ -105,7 +105,7 @@ func TestDefinitionIntoADependencyJarAndBack(t *testing.T) { write(t, root, map[string]string{"pom.xml": depsPOM, "src/main/java/com/acme/UseDeps.java": useDeps}) repo := mavenRepo(t) before := treeHash(t, repo) - w := New(Options{Root: root, State: filepath.Join(t.TempDir(), "state"), MavenRepo: repo, + w := New(Options{Root: root, State: filepath.Join(t.TempDir(), "state"), Dependencies: true, MavenRepo: repo, ListFiles: walkFiles, Freshness: -1, Logf: t.Logf}) doc := w.DocOf(filepath.Join(root, "src/main/java/com/acme/UseDeps.java")) @@ -178,7 +178,7 @@ func TestEditingPOMRereadsTheDependencies(t *testing.T) { root := t.TempDir() src := "package com.acme;\n\nimport com.example.helper.Helper;\n\nclass A {\n String s = Helper.shout(\"x\");\n}\n" write(t, root, map[string]string{"pom.xml": depsPOM, "src/main/java/com/acme/A.java": src}) - w := New(Options{Root: root, State: filepath.Join(t.TempDir(), "state"), MavenRepo: mavenRepo(t), + w := New(Options{Root: root, State: filepath.Join(t.TempDir(), "state"), Dependencies: true, MavenRepo: mavenRepo(t), ListFiles: walkFiles, Freshness: -1, Logf: t.Logf}) doc := w.DocOf(filepath.Join(root, "src/main/java/com/acme/A.java")) if _, ok, err := w.Definition(ctx(t), doc, []byte(src), at(t, src, "shout", 0, 1)); err != nil || ok { @@ -235,3 +235,43 @@ func TestGeneratedSourcesAreListed(t *testing.T) { t.Errorf("listed other build output: %v", files) } } + +// A project without Maven, its jars in WEB-INF/lib as an Ant or Eclipse web +// application keeps them, navigates into them; a jar added to lib/ is found +// by the next scan. +func TestDefinitionIntoJarsTheTreeKeeps(t *testing.T) { + root := t.TempDir() + src := "package com.acme;\n\nimport com.example.lib.Container;\nimport com.example.helper.Helper;\n\nclass A {\n Container box;\n String s = Helper.shout(\"x\");\n}\n" + write(t, root, map[string]string{"build.xml": "", "src/com/acme/A.java": src}) + fixture := filepath.Join("..", "deps", "testdata", "fixture") + if err := os.MkdirAll(filepath.Join(root, "WEB-INF", "lib"), 0o755); err != nil { + t.Fatal(err) + } + copyFile(t, filepath.Join(fixture, "lib-1.0.jar"), filepath.Join(root, "WEB-INF", "lib", "lib-1.0.jar")) + before := treeHash(t, filepath.Join(root, "WEB-INF")) + w := New(Options{Root: root, State: filepath.Join(t.TempDir(), "state"), Dependencies: true, + ListFiles: walkFiles, Freshness: -1, Logf: t.Logf}) + doc := w.DocOf(filepath.Join(root, "src/com/acme/A.java")) + tgt, ok, err := w.Definition(ctx(t), doc, []byte(src), at(t, src, "Container", 0, 1)) + if err != nil || !ok || !strings.Contains(filepath.ToSlash(tgt.File), "/state/dependency-sources/lib-1.0-") { + t.Fatalf("Container: %+v %v %v", tgt, ok, err) + } + if _, ok, err := w.Definition(ctx(t), doc, []byte(src), at(t, src, "shout", 0, 1)); err != nil || ok { + t.Fatalf("Helper resolved before its jar was added: %v %v", ok, err) + } + if err := os.MkdirAll(filepath.Join(root, "lib"), 0o755); err != nil { + t.Fatal(err) + } + copyFile(t, filepath.Join(fixture, "helper-2.0.jar"), filepath.Join(root, "lib", "helper-2.0.jar")) + copyFile(t, filepath.Join(fixture, "helper-2.0-sources.jar"), filepath.Join(root, "lib", "helper-2.0-sources.jar")) + tgt, ok, err = w.Definition(ctx(t), doc, []byte(src), at(t, src, "shout", 0, 1)) + if err != nil || !ok { + t.Fatalf("Helper.shout after the jar was added: %v %v", ok, err) + } + if b, _ := os.ReadFile(tgt.File); !strings.Contains(string(b), "return text.toUpperCase();") { + t.Errorf("shout did not open the sources jar:\n%s", b) + } + if treeHash(t, filepath.Join(root, "WEB-INF")) != before { + t.Errorf("WEB-INF changed") + } +} diff --git a/internal/workspace/workspace.go b/internal/workspace/workspace.go index 425093b..19523ba 100644 --- a/internal/workspace/workspace.go +++ b/internal/workspace/workspace.go @@ -70,8 +70,13 @@ type Options struct { State string // FindJDK locates the JDK whose sources are used; nil disables them. FindJDK func(context.Context) (jdk.JDK, bool) - // MavenRepo is the local Maven repository whose jars the folder's - // dependencies are read from; "" (or an empty State) disables them. + // Dependencies enables navigation into the folder's dependency jars; + // it needs State. They are the Maven dependencies its pom.xml files + // declare, read from MavenRepo, and the jars the tree keeps itself + // (deps.TreeJars). + Dependencies bool + // MavenRepo is the local Maven repository; "" leaves Maven dependencies + // out. MavenRepo string // ListFiles lists the files of root as slash-separated relative paths; // nil uses ListFiles. @@ -120,7 +125,7 @@ type Workspace struct { sources *jdk.Sources looked bool // the JDK was looked for deps *deps.Set // the dependency jars, nil until listed - poms map[string]stamp // the pom.xml files deps was read from + inputs map[string]stamp // the build files and tree jars deps was read from depsAt time.Time // when deps was read lib *codenav.Library // the JDK and deps, as the index looks names up } @@ -459,7 +464,7 @@ func (w *Workspace) runScan(run *scanRun) { started := time.Now() w.mu.Lock() old := w.stamps - oldPoms := w.poms + oldInputs := w.inputs depsAt := w.depsAt w.stale = false lookJDK := w.opt.FindJDK != nil && (!w.looked || w.sources != nil && w.sources.Changed()) @@ -487,16 +492,21 @@ func (w *Workspace) runScan(run *scanRun) { if err != nil { err = fmt.Errorf("list source files: %w", err) } - poms := w.pomStamps(all) - lookDeps := err == nil && w.opt.MavenRepo != "" && w.opt.State != "" && - (oldPoms == nil || !sameStamps(oldPoms, poms) || time.Since(depsAt) > depsRefresh) + var poms []string + var tree []deps.Jar + var inputs map[string]stamp + useDeps := err == nil && w.opt.Dependencies && w.opt.State != "" + if useDeps { + poms, tree, inputs = w.depInputs(all) + } + lookDeps := useDeps && (oldInputs == nil || !sameStamps(oldInputs, inputs) || time.Since(depsAt) > depsRefresh) var depSet *deps.Set var depsErr error depsDone := make(chan struct{}) if lookDeps { go func() { defer close(depsDone) - depSet, depsErr = w.readDeps(ctx, poms) + depSet, depsErr = w.readDeps(ctx, poms, tree) }() } else { close(depsDone) @@ -526,7 +536,7 @@ func (w *Workspace) runScan(run *scanRun) { if depsErr != nil { w.opt.Logf("dependency jars unavailable: %v", depsErr) } else { - w.deps, w.poms, w.depsAt = depSet, poms, started + w.deps, w.inputs, w.depsAt = depSet, inputs, started changed = true } } @@ -586,19 +596,38 @@ func (w *Workspace) library() *codenav.Library { }) } -// pomStamps returns the size and modification time of the folder's -// pom.xml files among all, the folder's files. -func (w *Workspace) pomStamps(all []string) map[string]stamp { - out := map[string]stamp{} +// depInputs returns the absolute paths of the folder's pom.xml files, the +// jars its tree keeps, and the stamps of both and of its Eclipse .classpath +// files: a change to any of them rereads the dependencies. all is the +// folder's files. +func (w *Workspace) depInputs(all []string) (poms []string, tree []deps.Jar, inputs map[string]stamp) { + inputs = map[string]stamp{} + note := func(p string) bool { + fi, err := os.Stat(p) + if err != nil || !fi.Mode().IsRegular() { + return false + } + inputs[p] = stamp{size: fi.Size(), mod: fi.ModTime()} + return true + } for _, p := range all { - if path.Base(p) != "pom.xml" { - continue + switch path.Base(p) { + case "pom.xml": + if w.opt.MavenRepo != "" && note(w.abs(p)) { + poms = append(poms, w.abs(p)) + } + case ".classpath": + note(w.abs(p)) } - if fi, err := os.Stat(w.abs(p)); err == nil && fi.Mode().IsRegular() { - out[p] = stamp{size: fi.Size(), mod: fi.ModTime()} + } + tree = deps.TreeJars(w.opt.Root, all, w.opt.MavenRepo) + for _, j := range tree { + note(j.Path) + if j.Sources != "" { + note(j.Sources) } } - return out + return poms, tree, inputs } func sameStamps(a, b map[string]stamp) bool { @@ -613,21 +642,21 @@ func sameStamps(a, b map[string]stamp) bool { return true } -// readDeps lists the jars of the Maven dependencies the poms declare. -func (w *Workspace) readDeps(ctx context.Context, poms map[string]stamp) (*deps.Set, error) { - if len(poms) == 0 { - return deps.Open(nil, w.opt.Logf), nil - } - files := make([]string, 0, len(poms)) - for p := range poms { - files = append(files, w.abs(p)) - } - jars, err := deps.MavenJars(ctx, files, w.opt.MavenRepo, w.opt.Logf) - if err != nil { - return nil, err +// readDeps lists the jars of the Maven dependencies the poms declare, then +// the jars the tree keeps. +func (w *Workspace) readDeps(ctx context.Context, poms []string, tree []deps.Jar) (*deps.Set, error) { + var jars []deps.Jar + if len(poms) > 0 { + var err error + if jars, err = deps.MavenJars(ctx, poms, w.opt.MavenRepo, w.opt.Logf); err != nil { + return nil, err + } } + maven := len(jars) + jars = append(jars, tree...) set := deps.Open(jars, w.opt.Logf) - w.opt.Logf("dependency jars: %d of %d resolved, %d classes", set.Len(), len(jars), set.Classes()) + w.opt.Logf("dependency jars: %d of %d resolved (%d Maven, %d in the working tree), %d classes", + set.Len(), len(jars), maven, len(tree), set.Classes()) return set, nil }