Let the GitLab sync leave out groups and projects

container/oracle/initdb.d/04_settings.sql

@@ -35,3 +35,4 @@ insert into SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('EMAIL_
 
 -- SIM Specific
 insert into SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('SECRET_STORE_FILE_PATH', 'secrets/sim.properties', 'STRING', 'Path to the secrets file.  If relative, then relative to app server user home directory', 'SECRETS', 1);
+insert into SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('GITLAB_SYNC_EXCLUDE_CSV', 'acc/scratch', 'CSV', 'CSV of GitLab paths the GitLab sync leaves out, each a group (with its subgroups and projects) or a project, such as acc/scratch or acc/ops/demo.  Not case sensitive.  To leave out none, set a path no project has.', 'SYNC', 1);

container/oracle/upgrade/2.4.0_gitlab_sync_exclude.sql

@@ -0,0 +1,8 @@
+-- Add the setting that lists the GitLab groups and projects the GitLab sync leaves out, starting
+-- with the acc/scratch group. Run once, as SIM_OWNER or a user that may insert into its tables.
+-- The row must match container/oracle/initdb.d/04_settings.sql. Until it runs, the sync leaves
+-- out nothing, as before.
+
+insert into SIM_OWNER.SETTING (KEY, VALUE, TYPE, DESCRIPTION, TAG, WEIGHT) values ('GITLAB_SYNC_EXCLUDE_CSV', 'acc/scratch', 'CSV', 'CSV of GitLab paths the GitLab sync leaves out, each a group (with its subgroups and projects) or a project, such as acc/scratch or acc/ops/demo.  Not case sensitive.  To leave out none, set a path no project has.', 'SYNC', 1);
+
+commit;

src/integration/java/org/jlab/sim/PagesIT.java

@@ -89,6 +89,14 @@ void onlyAdminsReadSetup(String username, int status) throws Exception {
     assertEquals(status, Sim.get("/setup/settings", Sim.token(username)).statusCode());
   }
 
+  @Test
+  void settingsListTheGitLabSyncExclusions() throws Exception {
+    String text = Sim.page("/setup/settings", Sim.token(Sim.ADMIN)).text();
+
+    assertTrue(text.contains("GITLAB_SYNC_EXCLUDE_CSV"), "the setting is listed");
+    assertTrue(text.contains("acc/scratch"), "it leaves out the scratch group by default");
+  }
+
   @Test
   void anonymousSetupGoesToLogin() throws Exception {
     HttpResponse<String> response = Sim.get("/setup/settings", null);

src/main/java/org/jlab/sim/business/service/SyncService.java

@@ -4,8 +4,10 @@
 import jakarta.ejb.Stateless;
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
+import jakarta.json.JsonArrayBuilder;
 import jakarta.json.JsonObject;
 import jakarta.json.JsonReader;
+import jakarta.json.JsonValue;
 import java.io.IOException;
 import java.io.StringReader;
 import java.net.URI;
@@ -22,6 +24,7 @@
 import org.jlab.sim.persistence.view.SoftwareDiff;
 import org.jlab.smoothness.business.exception.UserFriendlyException;
 import org.jlab.smoothness.business.service.JPAService;
+import org.jlab.smoothness.business.service.SettingsService;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Attributes;
 import org.jsoup.nodes.Document;
@@ -39,6 +42,9 @@ public SyncService() {
 
   static final String DESCRIPTION = "description";
 
+  /** The setting that lists the GitLab groups and projects a sync leaves out. */
+  static final String GITLAB_SYNC_EXCLUDE_CSV = "GITLAB_SYNC_EXCLUDE_CSV";
+
   @PermitAll
   public List<Software> fetch(Repository repository) throws UserFriendlyException {
     if (repository == null) {
@@ -65,8 +71,8 @@ public PaginatedResult(List<Software> softwareList, String nextUrl) {
     }
   }
 
-  private PaginatedResult paginatedGitLabQuery(String url, Repository repository)
-      throws UserFriendlyException {
+  private PaginatedResult paginatedGitLabQuery(
+      String url, Repository repository, List<String> excludedPaths) throws UserFriendlyException {
     String accessToken = SecretStore.get("GITLAB_ACCESS_TOKEN");
 
     if (accessToken == null) {
@@ -95,27 +101,86 @@ private PaginatedResult paginatedGitLabQuery(String url, Repository repository)
     }
 
     return new PaginatedResult(
-        parseGitLabProjects(response.body(), repository), nextUrl(response.headers()));
+        parseGitLabProjects(response.body(), repository, excludedPaths),
+        nextUrl(response.headers()));
   }
 
   /**
-   * Parses a page of GitLab's group projects API (a JSON array of projects).
+   * Parses a page of GitLab's group projects API (a JSON array of projects), leaving out the
+   * excluded projects.
    *
    * @param json The response body
    * @param repository The repository the software is in
+   * @param excludedPaths The excluded group and project paths, from {@link #excludedGitLabPaths}
    * @return The software, with its topics and archived state
    * @throws UserFriendlyException If the JSON doesn't parse or lacks a field
    */
-  static List<Software> parseGitLabProjects(String json, Repository repository)
-      throws UserFriendlyException {
+  static List<Software> parseGitLabProjects(
+      String json, Repository repository, List<String> excludedPaths) throws UserFriendlyException {
     try (StringReader stringReader = new StringReader(json);
         JsonReader jsonReader = Json.createReader(stringReader)) {
-      return parseProjects(jsonReader.readArray(), "web_url", repository);
+      JsonArrayBuilder included = Json.createArrayBuilder();
+
+      for (JsonValue value : jsonReader.readArray()) {
+        String path = value.asJsonObject().getString("path_with_namespace", null);
+
+        if (!isExcludedGitLabPath(path, excludedPaths)) {
+          included.add(value);
+        }
+      }
+
+      return parseProjects(included.build(), "web_url", repository);
     } catch (Exception e) {
       throw new UserFriendlyException("Could not parse JSON", e);
     }
   }
 
+  /**
+   * The GitLab paths a sync leaves out, from the {@value #GITLAB_SYNC_EXCLUDE_CSV} setting: each a
+   * group, with its subgroups and projects, or a project, such as {@code acc/scratch}.
+   *
+   * @param csv The setting's values, or null if there is no such setting
+   * @return The paths, lower case and without surrounding slashes or blanks
+   */
+  static List<String> excludedGitLabPaths(List<String> csv) {
+    List<String> paths = new ArrayList<>();
+
+    if (csv != null) {
+      for (String entry : csv) {
+        String path = entry.trim().replaceAll("^/+|/+$", "").toLowerCase(Locale.US);
+
+        if (!path.isEmpty()) {
+          paths.add(path);
+        }
+      }
+    }
+
+    return paths;
+  }
+
+  /**
+   * Whether a GitLab project is excluded: its path is an excluded path or inside one.
+   *
+   * @param pathWithNamespace The project's full path, such as {@code acc/scratch/test}, or null
+   * @param excludedPaths The excluded paths, from {@link #excludedGitLabPaths}
+   * @return true if the project is excluded
+   */
+  static boolean isExcludedGitLabPath(String pathWithNamespace, List<String> excludedPaths) {
+    if (pathWithNamespace == null) {
+      return false;
+    }
+
+    String path = pathWithNamespace.toLowerCase(Locale.US);
+
+    for (String excluded : excludedPaths) {
+      if (path.equals(excluded) || path.startsWith(excluded + "/")) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
   /**
    * Parses a page of GitHub's repository search API (a JSON object with the repositories in its
    * items).
@@ -232,8 +297,11 @@ private List<Software> fetchGitLab(Repository repository) throws UserFriendlyExc
     String url =
         "https://code.jlab.org/api/v4/groups/acc/projects?include_subgroups=Y&per_page=100";
 
+    List<String> excludedPaths =
+        excludedGitLabPaths(SettingsService.cachedSettings.csv(GITLAB_SYNC_EXCLUDE_CSV));
+
     do {
-      PaginatedResult result = paginatedGitLabQuery(url, repository);
+      PaginatedResult result = paginatedGitLabQuery(url, repository, excludedPaths);
       softwareList.addAll(result.softwareList);
       url = result.nextUrl;
     } while (url != null);

src/test/java/org/jlab/sim/business/service/SyncServiceParseTest.java

@@ -72,7 +72,7 @@ void gitLabProjectsBecomeSoftware() throws Exception {
         ]
         """;
 
-    List<Software> softwareList = SyncService.parseGitLabProjects(json, REPOSITORY);
+    List<Software> softwareList = SyncService.parseGitLabProjects(json, REPOSITORY, List.of());
 
     assertEquals(2, softwareList.size());
 
@@ -103,7 +103,7 @@ void multiLineDescriptionMatchesWhatSimSaved() throws Exception {
         ]
         """;
 
-    List<Software> remoteList = SyncService.parseGitLabProjects(json, REPOSITORY);
+    List<Software> remoteList = SyncService.parseGitLabProjects(json, REPOSITORY, List.of());
 
     assertEquals("FSD library\nfor apps\nand scripts\n", remoteList.get(0).getDescription());
 
@@ -124,12 +124,58 @@ void multiLineDescriptionMatchesWhatSimSaved() throws Exception {
     assertFalse(diff.hasChanges());
   }
 
+  @Test
+  void excludedGitLabGroupsAndProjectsAreLeftOut() throws Exception {
+    String json =
+        """
+        [
+          {"name": "fsdlib", "path_with_namespace": "acc/ops/fsdlib",
+           "web_url": "https://code.jlab.org/acc/ops/fsdlib", "description": null,
+           "topics": [], "archived": false},
+          {"name": "try-this", "path_with_namespace": "acc/scratch/try-this",
+           "web_url": "https://code.jlab.org/acc/scratch/try-this", "description": null,
+           "topics": [], "archived": false},
+          {"name": "deep", "path_with_namespace": "acc/Scratch/jdoe/deep",
+           "web_url": "https://code.jlab.org/acc/Scratch/jdoe/deep", "description": null,
+           "topics": [], "archived": false},
+          {"name": "scratchpad", "path_with_namespace": "acc/scratchpad",
+           "web_url": "https://code.jlab.org/acc/scratchpad", "description": null,
+           "topics": [], "archived": false},
+          {"name": "demo", "path_with_namespace": "acc/ops/demo",
+           "web_url": "https://code.jlab.org/acc/ops/demo", "description": null,
+           "topics": [], "archived": false}
+        ]
+        """;
+
+    List<String> excluded =
+        SyncService.excludedGitLabPaths(List.of("acc/scratch", " /ACC/ops/demo/ ", ""));
+
+    assertEquals(List.of("acc/scratch", "acc/ops/demo"), excluded, "trimmed and lower case");
+
+    List<Software> softwareList = SyncService.parseGitLabProjects(json, REPOSITORY, excluded);
+
+    assertEquals(
+        List.of("fsdlib", "scratchpad"),
+        softwareList.stream().map(Software::getName).toList(),
+        "a group's projects and subgroups are left out, as is a listed project, but not a"
+            + " sibling whose name starts the same");
+  }
+
+  @Test
+  void noExcludeSettingExcludesNothing() {
+    assertEquals(List.of(), SyncService.excludedGitLabPaths(null));
+    assertFalse(SyncService.isExcludedGitLabPath("acc/scratch/x", List.of()));
+    assertFalse(
+        SyncService.isExcludedGitLabPath(null, List.of("acc/scratch")),
+        "a project without a path is kept");
+  }
+
   @Test
   void emptyResultsAreNoSoftware() throws Exception {
     assertEquals(
         List.of(),
         SyncService.parseGitHubSearch("{\"total_count\": 0, \"items\": []}", REPOSITORY));
-    assertEquals(List.of(), SyncService.parseGitLabProjects("[]", REPOSITORY));
+    assertEquals(List.of(), SyncService.parseGitLabProjects("[]", REPOSITORY, List.of()));
   }
 
   @Test
@@ -145,7 +191,7 @@ void unexpectedJsonFails() {
               UserFriendlyException.class,
               () -> {
                 if (json.startsWith("[{")) {
-                  SyncService.parseGitLabProjects(json, REPOSITORY);
+                  SyncService.parseGitLabProjects(json, REPOSITORY, List.of());
                 } else {
                   SyncService.parseGitHubSearch(json, REPOSITORY);
                 }
@@ -156,7 +202,9 @@ void unexpectedJsonFails() {
 
     assertThrows(
         UserFriendlyException.class,
-        () -> SyncService.parseGitLabProjects("{\"message\": \"401 Unauthorized\"}", REPOSITORY));
+        () ->
+            SyncService.parseGitLabProjects(
+                "{\"message\": \"401 Unauthorized\"}", REPOSITORY, List.of()));
   }
 
   @Test