diff --git a/container/oracle/initdb.d/04_settings.sql b/container/oracle/initdb.d/04_settings.sql index 47184c0..d16e2f4 100644 --- a/container/oracle/initdb.d/04_settings.sql +++ b/container/oracle/initdb.d/04_settings.sql @@ -35,3 +35,4 @@ insert into SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('EMAIL_ -- SIM Specific insert into SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('SECRET_STORE_FILE_PATH', 'secrets/sim.properties', 'STRING', 'Path to the secrets file. If relative, then relative to app server user home directory', 'SECRETS', 1); +insert into SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('GITLAB_SYNC_EXCLUDE_CSV', 'acc/scratch', 'CSV', 'CSV of GitLab paths the GitLab sync leaves out, each a group (with its subgroups and projects) or a project, such as acc/scratch or acc/ops/demo. Not case sensitive. To leave out none, set a path no project has.', 'SYNC', 1); diff --git a/container/oracle/upgrade/2.4.0_gitlab_sync_exclude.sql b/container/oracle/upgrade/2.4.0_gitlab_sync_exclude.sql new file mode 100644 index 0000000..7d94d98 --- /dev/null +++ b/container/oracle/upgrade/2.4.0_gitlab_sync_exclude.sql @@ -0,0 +1,8 @@ +-- Add the setting that lists the GitLab groups and projects the GitLab sync leaves out, starting +-- with the acc/scratch group. Run once, as SIM_OWNER or a user that may insert into its tables. +-- The row must match container/oracle/initdb.d/04_settings.sql. Until it runs, the sync leaves +-- out nothing, as before. + +insert into SIM_OWNER.SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('GITLAB_SYNC_EXCLUDE_CSV', 'acc/scratch', 'CSV', 'CSV of GitLab paths the GitLab sync leaves out, each a group (with its subgroups and projects) or a project, such as acc/scratch or acc/ops/demo. Not case sensitive. To leave out none, set a path no project has.', 'SYNC', 1); + +commit; diff --git a/src/integration/java/org/jlab/sim/PagesIT.java b/src/integration/java/org/jlab/sim/PagesIT.java index 99097a5..9bacf5d 100644 --- a/src/integration/java/org/jlab/sim/PagesIT.java +++ b/src/integration/java/org/jlab/sim/PagesIT.java @@ -89,6 +89,14 @@ void onlyAdminsReadSetup(String username, int status) throws Exception { assertEquals(status, Sim.get("/setup/settings", Sim.token(username)).statusCode()); } + @Test + void settingsListTheGitLabSyncExclusions() throws Exception { + String text = Sim.page("/setup/settings", Sim.token(Sim.ADMIN)).text(); + + assertTrue(text.contains("GITLAB_SYNC_EXCLUDE_CSV"), "the setting is listed"); + assertTrue(text.contains("acc/scratch"), "it leaves out the scratch group by default"); + } + @Test void anonymousSetupGoesToLogin() throws Exception { HttpResponse response = Sim.get("/setup/settings", null); diff --git a/src/main/java/org/jlab/sim/business/service/SyncService.java b/src/main/java/org/jlab/sim/business/service/SyncService.java index 84a9894..0380901 100644 --- a/src/main/java/org/jlab/sim/business/service/SyncService.java +++ b/src/main/java/org/jlab/sim/business/service/SyncService.java @@ -4,8 +4,10 @@ import jakarta.ejb.Stateless; import jakarta.json.Json; import jakarta.json.JsonArray; +import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; import jakarta.json.JsonReader; +import jakarta.json.JsonValue; import java.io.IOException; import java.io.StringReader; import java.net.URI; @@ -22,6 +24,7 @@ import org.jlab.sim.persistence.view.SoftwareDiff; import org.jlab.smoothness.business.exception.UserFriendlyException; import org.jlab.smoothness.business.service.JPAService; +import org.jlab.smoothness.business.service.SettingsService; import org.jsoup.Jsoup; import org.jsoup.nodes.Attributes; import org.jsoup.nodes.Document; @@ -39,6 +42,9 @@ public SyncService() { static final String DESCRIPTION = "description"; + /** The setting that lists the GitLab groups and projects a sync leaves out. */ + static final String GITLAB_SYNC_EXCLUDE_CSV = "GITLAB_SYNC_EXCLUDE_CSV"; + @PermitAll public List fetch(Repository repository) throws UserFriendlyException { if (repository == null) { @@ -65,8 +71,8 @@ public PaginatedResult(List softwareList, String nextUrl) { } } - private PaginatedResult paginatedGitLabQuery(String url, Repository repository) - throws UserFriendlyException { + private PaginatedResult paginatedGitLabQuery( + String url, Repository repository, List excludedPaths) throws UserFriendlyException { String accessToken = SecretStore.get("GITLAB_ACCESS_TOKEN"); if (accessToken == null) { @@ -95,27 +101,86 @@ private PaginatedResult paginatedGitLabQuery(String url, Repository repository) } return new PaginatedResult( - parseGitLabProjects(response.body(), repository), nextUrl(response.headers())); + parseGitLabProjects(response.body(), repository, excludedPaths), + nextUrl(response.headers())); } /** - * Parses a page of GitLab's group projects API (a JSON array of projects). + * Parses a page of GitLab's group projects API (a JSON array of projects), leaving out the + * excluded projects. * * @param json The response body * @param repository The repository the software is in + * @param excludedPaths The excluded group and project paths, from {@link #excludedGitLabPaths} * @return The software, with its topics and archived state * @throws UserFriendlyException If the JSON doesn't parse or lacks a field */ - static List parseGitLabProjects(String json, Repository repository) - throws UserFriendlyException { + static List parseGitLabProjects( + String json, Repository repository, List excludedPaths) throws UserFriendlyException { try (StringReader stringReader = new StringReader(json); JsonReader jsonReader = Json.createReader(stringReader)) { - return parseProjects(jsonReader.readArray(), "web_url", repository); + JsonArrayBuilder included = Json.createArrayBuilder(); + + for (JsonValue value : jsonReader.readArray()) { + String path = value.asJsonObject().getString("path_with_namespace", null); + + if (!isExcludedGitLabPath(path, excludedPaths)) { + included.add(value); + } + } + + return parseProjects(included.build(), "web_url", repository); } catch (Exception e) { throw new UserFriendlyException("Could not parse JSON", e); } } + /** + * The GitLab paths a sync leaves out, from the {@value #GITLAB_SYNC_EXCLUDE_CSV} setting: each a + * group, with its subgroups and projects, or a project, such as {@code acc/scratch}. + * + * @param csv The setting's values, or null if there is no such setting + * @return The paths, lower case and without surrounding slashes or blanks + */ + static List excludedGitLabPaths(List csv) { + List paths = new ArrayList<>(); + + if (csv != null) { + for (String entry : csv) { + String path = entry.trim().replaceAll("^/+|/+$", "").toLowerCase(Locale.US); + + if (!path.isEmpty()) { + paths.add(path); + } + } + } + + return paths; + } + + /** + * Whether a GitLab project is excluded: its path is an excluded path or inside one. + * + * @param pathWithNamespace The project's full path, such as {@code acc/scratch/test}, or null + * @param excludedPaths The excluded paths, from {@link #excludedGitLabPaths} + * @return true if the project is excluded + */ + static boolean isExcludedGitLabPath(String pathWithNamespace, List excludedPaths) { + if (pathWithNamespace == null) { + return false; + } + + String path = pathWithNamespace.toLowerCase(Locale.US); + + for (String excluded : excludedPaths) { + if (path.equals(excluded) || path.startsWith(excluded + "/")) { + return true; + } + } + + return false; + } + /** * Parses a page of GitHub's repository search API (a JSON object with the repositories in its * items). @@ -232,8 +297,11 @@ private List fetchGitLab(Repository repository) throws UserFriendlyExc String url = "https://code.jlab.org/api/v4/groups/acc/projects?include_subgroups=Y&per_page=100"; + List excludedPaths = + excludedGitLabPaths(SettingsService.cachedSettings.csv(GITLAB_SYNC_EXCLUDE_CSV)); + do { - PaginatedResult result = paginatedGitLabQuery(url, repository); + PaginatedResult result = paginatedGitLabQuery(url, repository, excludedPaths); softwareList.addAll(result.softwareList); url = result.nextUrl; } while (url != null); diff --git a/src/test/java/org/jlab/sim/business/service/SyncServiceParseTest.java b/src/test/java/org/jlab/sim/business/service/SyncServiceParseTest.java index a26d507..b358300 100644 --- a/src/test/java/org/jlab/sim/business/service/SyncServiceParseTest.java +++ b/src/test/java/org/jlab/sim/business/service/SyncServiceParseTest.java @@ -72,7 +72,7 @@ void gitLabProjectsBecomeSoftware() throws Exception { ] """; - List softwareList = SyncService.parseGitLabProjects(json, REPOSITORY); + List softwareList = SyncService.parseGitLabProjects(json, REPOSITORY, List.of()); assertEquals(2, softwareList.size()); @@ -103,7 +103,7 @@ void multiLineDescriptionMatchesWhatSimSaved() throws Exception { ] """; - List remoteList = SyncService.parseGitLabProjects(json, REPOSITORY); + List remoteList = SyncService.parseGitLabProjects(json, REPOSITORY, List.of()); assertEquals("FSD library\nfor apps\nand scripts\n", remoteList.get(0).getDescription()); @@ -124,12 +124,58 @@ void multiLineDescriptionMatchesWhatSimSaved() throws Exception { assertFalse(diff.hasChanges()); } + @Test + void excludedGitLabGroupsAndProjectsAreLeftOut() throws Exception { + String json = + """ + [ + {"name": "fsdlib", "path_with_namespace": "acc/ops/fsdlib", + "web_url": "https://code.jlab.org/acc/ops/fsdlib", "description": null, + "topics": [], "archived": false}, + {"name": "try-this", "path_with_namespace": "acc/scratch/try-this", + "web_url": "https://code.jlab.org/acc/scratch/try-this", "description": null, + "topics": [], "archived": false}, + {"name": "deep", "path_with_namespace": "acc/Scratch/jdoe/deep", + "web_url": "https://code.jlab.org/acc/Scratch/jdoe/deep", "description": null, + "topics": [], "archived": false}, + {"name": "scratchpad", "path_with_namespace": "acc/scratchpad", + "web_url": "https://code.jlab.org/acc/scratchpad", "description": null, + "topics": [], "archived": false}, + {"name": "demo", "path_with_namespace": "acc/ops/demo", + "web_url": "https://code.jlab.org/acc/ops/demo", "description": null, + "topics": [], "archived": false} + ] + """; + + List excluded = + SyncService.excludedGitLabPaths(List.of("acc/scratch", " /ACC/ops/demo/ ", "")); + + assertEquals(List.of("acc/scratch", "acc/ops/demo"), excluded, "trimmed and lower case"); + + List softwareList = SyncService.parseGitLabProjects(json, REPOSITORY, excluded); + + assertEquals( + List.of("fsdlib", "scratchpad"), + softwareList.stream().map(Software::getName).toList(), + "a group's projects and subgroups are left out, as is a listed project, but not a" + + " sibling whose name starts the same"); + } + + @Test + void noExcludeSettingExcludesNothing() { + assertEquals(List.of(), SyncService.excludedGitLabPaths(null)); + assertFalse(SyncService.isExcludedGitLabPath("acc/scratch/x", List.of())); + assertFalse( + SyncService.isExcludedGitLabPath(null, List.of("acc/scratch")), + "a project without a path is kept"); + } + @Test void emptyResultsAreNoSoftware() throws Exception { assertEquals( List.of(), SyncService.parseGitHubSearch("{\"total_count\": 0, \"items\": []}", REPOSITORY)); - assertEquals(List.of(), SyncService.parseGitLabProjects("[]", REPOSITORY)); + assertEquals(List.of(), SyncService.parseGitLabProjects("[]", REPOSITORY, List.of())); } @Test @@ -145,7 +191,7 @@ void unexpectedJsonFails() { UserFriendlyException.class, () -> { if (json.startsWith("[{")) { - SyncService.parseGitLabProjects(json, REPOSITORY); + SyncService.parseGitLabProjects(json, REPOSITORY, List.of()); } else { SyncService.parseGitHubSearch(json, REPOSITORY); } @@ -156,7 +202,9 @@ void unexpectedJsonFails() { assertThrows( UserFriendlyException.class, - () -> SyncService.parseGitLabProjects("{\"message\": \"401 Unauthorized\"}", REPOSITORY)); + () -> + SyncService.parseGitLabProjects( + "{\"message\": \"401 Unauthorized\"}", REPOSITORY, List.of())); } @Test