From 82fc07a47a6889e630ef9e230f23b61a20655c28 Mon Sep 17 00:00:00 2001 From: "slominskir-coding-agent[bot]" <335168828+slominskir-coding-agent[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:23:27 -0500 Subject: [PATCH] Let the GitLab sync leave out groups and projects The GitLab sync reads every project in the acc group and its subgroups, so it suggests adding the throwaway projects in acc/scratch. A new CSV setting, GITLAB_SYNC_EXCLUDE_CSV, lists GitLab paths the sync leaves out: a group with its subgroups and projects, or a single project, matched on the project's path_with_namespace without regard to case or surrounding slashes. It defaults to acc/scratch. Existing databases get the setting from container/oracle/upgrade/2.4.0_gitlab_sync_exclude.sql; without it the sync leaves out nothing, as before. Co-Authored-By: Claude Opus 5.5 --- container/oracle/initdb.d/04_settings.sql | 1 + .../upgrade/2.4.0_gitlab_sync_exclude.sql | 8 ++ .../java/org/jlab/sim/PagesIT.java | 8 ++ .../sim/business/service/SyncService.java | 84 +++++++++++++++++-- .../service/SyncServiceParseTest.java | 58 +++++++++++-- 5 files changed, 146 insertions(+), 13 deletions(-) create mode 100644 container/oracle/upgrade/2.4.0_gitlab_sync_exclude.sql diff --git a/container/oracle/initdb.d/04_settings.sql b/container/oracle/initdb.d/04_settings.sql index 47184c0..d16e2f4 100644 --- a/container/oracle/initdb.d/04_settings.sql +++ b/container/oracle/initdb.d/04_settings.sql @@ -35,3 +35,4 @@ insert into SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('EMAIL_ -- SIM Specific insert into SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('SECRET_STORE_FILE_PATH', 'secrets/sim.properties', 'STRING', 'Path to the secrets file. If relative, then relative to app server user home directory', 'SECRETS', 1); +insert into SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('GITLAB_SYNC_EXCLUDE_CSV', 'acc/scratch', 'CSV', 'CSV of GitLab paths the GitLab sync leaves out, each a group (with its subgroups and projects) or a project, such as acc/scratch or acc/ops/demo. Not case sensitive. To leave out none, set a path no project has.', 'SYNC', 1); diff --git a/container/oracle/upgrade/2.4.0_gitlab_sync_exclude.sql b/container/oracle/upgrade/2.4.0_gitlab_sync_exclude.sql new file mode 100644 index 0000000..7d94d98 --- /dev/null +++ b/container/oracle/upgrade/2.4.0_gitlab_sync_exclude.sql @@ -0,0 +1,8 @@ +-- Add the setting that lists the GitLab groups and projects the GitLab sync leaves out, starting +-- with the acc/scratch group. Run once, as SIM_OWNER or a user that may insert into its tables. +-- The row must match container/oracle/initdb.d/04_settings.sql. Until it runs, the sync leaves +-- out nothing, as before. + +insert into SIM_OWNER.SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('GITLAB_SYNC_EXCLUDE_CSV', 'acc/scratch', 'CSV', 'CSV of GitLab paths the GitLab sync leaves out, each a group (with its subgroups and projects) or a project, such as acc/scratch or acc/ops/demo. Not case sensitive. To leave out none, set a path no project has.', 'SYNC', 1); + +commit; diff --git a/src/integration/java/org/jlab/sim/PagesIT.java b/src/integration/java/org/jlab/sim/PagesIT.java index 99097a5..9bacf5d 100644 --- a/src/integration/java/org/jlab/sim/PagesIT.java +++ b/src/integration/java/org/jlab/sim/PagesIT.java @@ -89,6 +89,14 @@ void onlyAdminsReadSetup(String username, int status) throws Exception { assertEquals(status, Sim.get("/setup/settings", Sim.token(username)).statusCode()); } + @Test + void settingsListTheGitLabSyncExclusions() throws Exception { + String text = Sim.page("/setup/settings", Sim.token(Sim.ADMIN)).text(); + + assertTrue(text.contains("GITLAB_SYNC_EXCLUDE_CSV"), "the setting is listed"); + assertTrue(text.contains("acc/scratch"), "it leaves out the scratch group by default"); + } + @Test void anonymousSetupGoesToLogin() throws Exception { HttpResponse response = Sim.get("/setup/settings", null); diff --git a/src/main/java/org/jlab/sim/business/service/SyncService.java b/src/main/java/org/jlab/sim/business/service/SyncService.java index 84a9894..0380901 100644 --- a/src/main/java/org/jlab/sim/business/service/SyncService.java +++ b/src/main/java/org/jlab/sim/business/service/SyncService.java @@ -4,8 +4,10 @@ import jakarta.ejb.Stateless; import jakarta.json.Json; import jakarta.json.JsonArray; +import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; import jakarta.json.JsonReader; +import jakarta.json.JsonValue; import java.io.IOException; import java.io.StringReader; import java.net.URI; @@ -22,6 +24,7 @@ import org.jlab.sim.persistence.view.SoftwareDiff; import org.jlab.smoothness.business.exception.UserFriendlyException; import org.jlab.smoothness.business.service.JPAService; +import org.jlab.smoothness.business.service.SettingsService; import org.jsoup.Jsoup; import org.jsoup.nodes.Attributes; import org.jsoup.nodes.Document; @@ -39,6 +42,9 @@ public SyncService() { static final String DESCRIPTION = "description"; + /** The setting that lists the GitLab groups and projects a sync leaves out. */ + static final String GITLAB_SYNC_EXCLUDE_CSV = "GITLAB_SYNC_EXCLUDE_CSV"; + @PermitAll public List fetch(Repository repository) throws UserFriendlyException { if (repository == null) { @@ -65,8 +71,8 @@ public PaginatedResult(List softwareList, String nextUrl) { } } - private PaginatedResult paginatedGitLabQuery(String url, Repository repository) - throws UserFriendlyException { + private PaginatedResult paginatedGitLabQuery( + String url, Repository repository, List excludedPaths) throws UserFriendlyException { String accessToken = SecretStore.get("GITLAB_ACCESS_TOKEN"); if (accessToken == null) { @@ -95,27 +101,86 @@ private PaginatedResult paginatedGitLabQuery(String url, Repository repository) } return new PaginatedResult( - parseGitLabProjects(response.body(), repository), nextUrl(response.headers())); + parseGitLabProjects(response.body(), repository, excludedPaths), + nextUrl(response.headers())); } /** - * Parses a page of GitLab's group projects API (a JSON array of projects). + * Parses a page of GitLab's group projects API (a JSON array of projects), leaving out the + * excluded projects. * * @param json The response body * @param repository The repository the software is in + * @param excludedPaths The excluded group and project paths, from {@link #excludedGitLabPaths} * @return The software, with its topics and archived state * @throws UserFriendlyException If the JSON doesn't parse or lacks a field */ - static List parseGitLabProjects(String json, Repository repository) - throws UserFriendlyException { + static List parseGitLabProjects( + String json, Repository repository, List excludedPaths) throws UserFriendlyException { try (StringReader stringReader = new StringReader(json); JsonReader jsonReader = Json.createReader(stringReader)) { - return parseProjects(jsonReader.readArray(), "web_url", repository); + JsonArrayBuilder included = Json.createArrayBuilder(); + + for (JsonValue value : jsonReader.readArray()) { + String path = value.asJsonObject().getString("path_with_namespace", null); + + if (!isExcludedGitLabPath(path, excludedPaths)) { + included.add(value); + } + } + + return parseProjects(included.build(), "web_url", repository); } catch (Exception e) { throw new UserFriendlyException("Could not parse JSON", e); } } + /** + * The GitLab paths a sync leaves out, from the {@value #GITLAB_SYNC_EXCLUDE_CSV} setting: each a + * group, with its subgroups and projects, or a project, such as {@code acc/scratch}. + * + * @param csv The setting's values, or null if there is no such setting + * @return The paths, lower case and without surrounding slashes or blanks + */ + static List excludedGitLabPaths(List csv) { + List paths = new ArrayList<>(); + + if (csv != null) { + for (String entry : csv) { + String path = entry.trim().replaceAll("^/+|/+$", "").toLowerCase(Locale.US); + + if (!path.isEmpty()) { + paths.add(path); + } + } + } + + return paths; + } + + /** + * Whether a GitLab project is excluded: its path is an excluded path or inside one. + * + * @param pathWithNamespace The project's full path, such as {@code acc/scratch/test}, or null + * @param excludedPaths The excluded paths, from {@link #excludedGitLabPaths} + * @return true if the project is excluded + */ + static boolean isExcludedGitLabPath(String pathWithNamespace, List excludedPaths) { + if (pathWithNamespace == null) { + return false; + } + + String path = pathWithNamespace.toLowerCase(Locale.US); + + for (String excluded : excludedPaths) { + if (path.equals(excluded) || path.startsWith(excluded + "/")) { + return true; + } + } + + return false; + } + /** * Parses a page of GitHub's repository search API (a JSON object with the repositories in its * items). @@ -232,8 +297,11 @@ private List fetchGitLab(Repository repository) throws UserFriendlyExc String url = "https://code.jlab.org/api/v4/groups/acc/projects?include_subgroups=Y&per_page=100"; + List excludedPaths = + excludedGitLabPaths(SettingsService.cachedSettings.csv(GITLAB_SYNC_EXCLUDE_CSV)); + do { - PaginatedResult result = paginatedGitLabQuery(url, repository); + PaginatedResult result = paginatedGitLabQuery(url, repository, excludedPaths); softwareList.addAll(result.softwareList); url = result.nextUrl; } while (url != null); diff --git a/src/test/java/org/jlab/sim/business/service/SyncServiceParseTest.java b/src/test/java/org/jlab/sim/business/service/SyncServiceParseTest.java index a26d507..b358300 100644 --- a/src/test/java/org/jlab/sim/business/service/SyncServiceParseTest.java +++ b/src/test/java/org/jlab/sim/business/service/SyncServiceParseTest.java @@ -72,7 +72,7 @@ void gitLabProjectsBecomeSoftware() throws Exception { ] """; - List softwareList = SyncService.parseGitLabProjects(json, REPOSITORY); + List softwareList = SyncService.parseGitLabProjects(json, REPOSITORY, List.of()); assertEquals(2, softwareList.size()); @@ -103,7 +103,7 @@ void multiLineDescriptionMatchesWhatSimSaved() throws Exception { ] """; - List remoteList = SyncService.parseGitLabProjects(json, REPOSITORY); + List remoteList = SyncService.parseGitLabProjects(json, REPOSITORY, List.of()); assertEquals("FSD library\nfor apps\nand scripts\n", remoteList.get(0).getDescription()); @@ -124,12 +124,58 @@ void multiLineDescriptionMatchesWhatSimSaved() throws Exception { assertFalse(diff.hasChanges()); } + @Test + void excludedGitLabGroupsAndProjectsAreLeftOut() throws Exception { + String json = + """ + [ + {"name": "fsdlib", "path_with_namespace": "acc/ops/fsdlib", + "web_url": "https://code.jlab.org/acc/ops/fsdlib", "description": null, + "topics": [], "archived": false}, + {"name": "try-this", "path_with_namespace": "acc/scratch/try-this", + "web_url": "https://code.jlab.org/acc/scratch/try-this", "description": null, + "topics": [], "archived": false}, + {"name": "deep", "path_with_namespace": "acc/Scratch/jdoe/deep", + "web_url": "https://code.jlab.org/acc/Scratch/jdoe/deep", "description": null, + "topics": [], "archived": false}, + {"name": "scratchpad", "path_with_namespace": "acc/scratchpad", + "web_url": "https://code.jlab.org/acc/scratchpad", "description": null, + "topics": [], "archived": false}, + {"name": "demo", "path_with_namespace": "acc/ops/demo", + "web_url": "https://code.jlab.org/acc/ops/demo", "description": null, + "topics": [], "archived": false} + ] + """; + + List excluded = + SyncService.excludedGitLabPaths(List.of("acc/scratch", " /ACC/ops/demo/ ", "")); + + assertEquals(List.of("acc/scratch", "acc/ops/demo"), excluded, "trimmed and lower case"); + + List softwareList = SyncService.parseGitLabProjects(json, REPOSITORY, excluded); + + assertEquals( + List.of("fsdlib", "scratchpad"), + softwareList.stream().map(Software::getName).toList(), + "a group's projects and subgroups are left out, as is a listed project, but not a" + + " sibling whose name starts the same"); + } + + @Test + void noExcludeSettingExcludesNothing() { + assertEquals(List.of(), SyncService.excludedGitLabPaths(null)); + assertFalse(SyncService.isExcludedGitLabPath("acc/scratch/x", List.of())); + assertFalse( + SyncService.isExcludedGitLabPath(null, List.of("acc/scratch")), + "a project without a path is kept"); + } + @Test void emptyResultsAreNoSoftware() throws Exception { assertEquals( List.of(), SyncService.parseGitHubSearch("{\"total_count\": 0, \"items\": []}", REPOSITORY)); - assertEquals(List.of(), SyncService.parseGitLabProjects("[]", REPOSITORY)); + assertEquals(List.of(), SyncService.parseGitLabProjects("[]", REPOSITORY, List.of())); } @Test @@ -145,7 +191,7 @@ void unexpectedJsonFails() { UserFriendlyException.class, () -> { if (json.startsWith("[{")) { - SyncService.parseGitLabProjects(json, REPOSITORY); + SyncService.parseGitLabProjects(json, REPOSITORY, List.of()); } else { SyncService.parseGitHubSearch(json, REPOSITORY); } @@ -156,7 +202,9 @@ void unexpectedJsonFails() { assertThrows( UserFriendlyException.class, - () -> SyncService.parseGitLabProjects("{\"message\": \"401 Unauthorized\"}", REPOSITORY)); + () -> + SyncService.parseGitLabProjects( + "{\"message\": \"401 Unauthorized\"}", REPOSITORY, List.of())); } @Test