From e35411c8feb816fbda7a34b3aa9e798673f220d5 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sat, 13 Jun 2026 20:55:08 -0700 Subject: [PATCH 01/24] Fixed Panorama Public reminder scheduling and hardened NCBI publication search - PanoramaPublicModule.startupAfterSpringConfig now re-establishes the daily reminder's Quartz schedule on startup; it was only set when an admin saved the settings form, so a Tomcat restart silently killed the job (Quartz's in-memory job store does not survive a JVM restart). * Added an "NCBI API key" admin field wired into buildCommonParams as &api_key= (raises NCBI rate limit 3->10 req/sec); contextual Logger threaded through getString/getJson so retry warnings follow the job log vs. server log. - NcbiPublicationSearchServiceImpl.getString retries eutils calls (3 attempts, 500/1000ms backoff) on 5xx and read timeouts (~40% transient failure rate); 4xx fails fast, with the NCBI response body included so a bad key's "API key invalid" reaches the log instead of a bare 400. - Added JUnit (retry/backoff, api_key, 4xx body) and Selenium API-key coverage. Follow-up to PR #606. --- .../PanoramaPublicController.java | 13 ++ .../panoramapublic/PanoramaPublicModule.java | 4 + .../message/PrivateDataReminderSettings.java | 15 ++ .../MockNcbiPublicationSearchService.java | 5 +- .../NcbiPublicationSearchServiceImpl.java | 191 ++++++++++++++++-- .../view/privateDataRemindersSettingsForm.jsp | 13 ++ .../PanoramaPublicBaseTest.java | 19 +- .../panoramapublic/PublicationSearchTest.java | 40 +++- 8 files changed, 277 insertions(+), 23 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java index cef3ae75..a9dc4619 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java @@ -10146,6 +10146,7 @@ public ModelAndView getView(PrivateDataReminderSettingsForm form, boolean reshow form.setExtensionLength(settings.getExtensionLength()); form.setEnablePublicationSearch(settings.isEnablePublicationSearch()); form.setPublicationSearchFrequency(settings.getPublicationSearchFrequency()); + form.setNcbiApiKey(settings.getNcbiApiKey()); } VBox view = new VBox(); @@ -10166,6 +10167,7 @@ public boolean handlePost(PrivateDataReminderSettingsForm form, BindException er settings.setExtensionLength(form.getExtensionLength()); settings.setEnablePublicationSearch(form.isEnablePublicationSearch()); settings.setPublicationSearchFrequency(form.getPublicationSearchFrequency()); + settings.setNcbiApiKey(form.getNcbiApiKey()); PrivateDataReminderSettings.save(settings); PrivateDataMessageScheduler.getInstance().initialize(settings.isEnableReminders()); @@ -10205,6 +10207,7 @@ public static class PrivateDataReminderSettingsForm private Integer _delayUntilFirstReminder; private boolean _enablePublicationSearch; private Integer _publicationSearchFrequency; + private String _ncbiApiKey; public boolean isEnabled() { @@ -10275,6 +10278,16 @@ public void setPublicationSearchFrequency(Integer publicationSearchFrequency) { _publicationSearchFrequency = publicationSearchFrequency; } + + public String getNcbiApiKey() + { + return _ncbiApiKey; + } + + public void setNcbiApiKey(String ncbiApiKey) + { + _ncbiApiKey = ncbiApiKey; + } } @RequiresPermission(AdminOperationsPermission.class) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java index 1c9ea49b..c7df83fb 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java @@ -46,6 +46,7 @@ import org.labkey.panoramapublic.bluesky.BlueskyApiClient; import org.labkey.panoramapublic.bluesky.PanoramaPublicLogoResourceType; import org.labkey.panoramapublic.catalog.CatalogImageAttachmentType; +import org.labkey.panoramapublic.message.PrivateDataMessageScheduler; import org.labkey.panoramapublic.message.PrivateDataReminderSettings; import org.labkey.panoramapublic.ncbi.NcbiPublicationSearchServiceImpl; import org.labkey.panoramapublic.model.Journal; @@ -151,6 +152,9 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) { fileContentService.addFileListener(new PanoramaPublicFileListener()); } + + // Start the private data reminder job on server restart if it is enabled. + PrivateDataMessageScheduler.getInstance().initialize(PrivateDataReminderSettings.get().isEnableReminders()); } @NotNull diff --git a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java index a75b411c..42cfd43b 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java +++ b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java @@ -41,6 +41,7 @@ public class PrivateDataReminderSettings public static final String PROP_EXTENSION_LENGTH = "Extension duration (months)"; public static final String PROP_ENABLE_PUBLICATION_SEARCH = "Enable publication search"; public static final String PROP_PUBLICATION_SEARCH_FREQUENCY = "Publication search frequency (months)"; + public static final String PROP_NCBI_API_KEY = "NCBI API key"; private static final boolean DEFAULT_ENABLE_REMINDERS = false; public static final String DEFAULT_REMINDER_TIME = "8:00 AM"; @@ -61,6 +62,7 @@ public class PrivateDataReminderSettings private int _extensionLength; private boolean _enablePublicationSearch; private int _publicationSearchFrequency; + private String _ncbiApiKey; public static PrivateDataReminderSettings get() { @@ -101,6 +103,8 @@ public static PrivateDataReminderSettings get() ? DEFAULT_PUBLICATION_SEARCH_FREQUENCY : Integer.valueOf(settingsMap.get(PROP_PUBLICATION_SEARCH_FREQUENCY)); settings.setPublicationSearchFrequency(publicationSearchFrequency); + + settings.setNcbiApiKey(settingsMap.get(PROP_NCBI_API_KEY)); } else { @@ -147,6 +151,7 @@ public static void save(PrivateDataReminderSettings settings) settingsMap.put(PROP_REMINDER_TIME, settings.getReminderTimeFormatted()); settingsMap.put(PROP_ENABLE_PUBLICATION_SEARCH, String.valueOf(settings.isEnablePublicationSearch())); settingsMap.put(PROP_PUBLICATION_SEARCH_FREQUENCY, String.valueOf(settings.getPublicationSearchFrequency())); + settingsMap.put(PROP_NCBI_API_KEY, settings.getNcbiApiKey() != null ? settings.getNcbiApiKey() : ""); settingsMap.save(); } @@ -225,6 +230,16 @@ public void setPublicationSearchFrequency(int publicationSearchFrequency) _publicationSearchFrequency = publicationSearchFrequency; } + public @Nullable String getNcbiApiKey() + { + return _ncbiApiKey; + } + + public void setNcbiApiKey(@Nullable String ncbiApiKey) + { + _ncbiApiKey = ncbiApiKey; + } + public @Nullable Date getReminderValidUntilDate(@NotNull DatasetStatus status) { return status.getLastReminderDate() == null ? null : addMonths(status.getLastReminderDate(), getReminderFrequency()); diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java index b32c5864..b9995737 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java @@ -15,6 +15,7 @@ */ package org.labkey.panoramapublic.ncbi; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.json.JSONArray; import org.json.JSONObject; @@ -28,7 +29,7 @@ /** * Mock implementation of {@link NcbiPublicationSearchService} that returns canned data registered by tests. * Used by Selenium tests when running on TeamCity. - * Extends {@link NcbiPublicationSearchServiceImpl} and only overrides {@link #getString(String)}, + * Extends {@link NcbiPublicationSearchServiceImpl} and only overrides {@link #getString(String, Logger)}, * the single method that makes HTTP calls to NCBI. All search logic, filtering, author/title * verification, citation parsing, and priority filtering run through the real implementation code. * Tests register mock articles via {@link #register}, providing the database, ID, search key, @@ -132,7 +133,7 @@ public void register(String database, String id, String searchKey, * Handles ESearch, ESummary, and Citation Exporter URLs. */ @Override - protected String getString(String url) throws IOException + protected String getString(String url, Logger log) throws IOException { if (url.contains("esearch.fcgi")) { diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index e6b0bae3..3f6fd727 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -38,10 +38,12 @@ import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.logging.LogHelper; import org.labkey.panoramapublic.datacite.DataCiteService; +import org.labkey.panoramapublic.message.PrivateDataReminderSettings; import org.labkey.panoramapublic.model.ExperimentAnnotations; import org.labkey.panoramapublic.ncbi.NcbiConstants.DB; import java.io.IOException; +import java.net.SocketTimeoutException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.text.Normalizer; @@ -95,6 +97,11 @@ public static void setInstance(NcbiPublicationSearchService impl) private static final int RATE_LIMIT_DELAY_MS = 400; // NCBI allows 3 requests/sec private static final int TIMEOUT_MS = 10000; // 10 seconds + // NCBI eutils intermittently returns transient 5xx errors and read timeouts, even well under + // the rate limit. Retry those a few times with exponential backoff before giving up. + private static final int MAX_HTTP_ATTEMPTS = 3; // initial try + 2 retries + private static final int RETRY_BASE_DELAY_MS = 500; // exponential backoff base + // NCBI suggests using the 'tool' and 'email' parameters on E-utilities URLs // https://www.nlm.nih.gov/dataguide/eutilities/utilities.html private static final String TOOL = "PanoramaPublic"; @@ -156,7 +163,7 @@ private static Logger getLog(@Nullable Logger logger) try { - String response = getString(queryUrl); + String response = getString(queryUrl, log); return parseCitation(response, publicationId, database, log); } catch (IOException e) @@ -326,7 +333,7 @@ private List executeSearch(String query, String database, Logger log) try { - JSONObject json = getJson(url); + JSONObject json = getJson(url, log); JSONObject eSearchResult = json.getJSONObject("esearchresult"); JSONArray idList = eSearchResult.getJSONArray("idlist"); @@ -349,16 +356,42 @@ private List executeSearch(String query, String database, Logger log) * @throws IOException if the request fails or the server returns a non-2xx response * @throws JSONException if the response body is not valid JSON */ - protected JSONObject getJson(String url) throws IOException + protected JSONObject getJson(String url, Logger log) throws IOException { - return new JSONObject(getString(url)); + return new JSONObject(getString(url, log)); } /** - * Execute an HTTP GET request and return the response body as a string. + * Execute an HTTP GET request and return the response body as a string. Retry warnings are + * written to {@code log} so they land in the pipeline job log when invoked from the reminder + * job (and in the server log for UI-triggered searches, which pass the static logger). * @throws IOException if the request fails or the server returns a non-2xx response */ - protected String getString(String url) throws IOException + protected String getString(String url, Logger log) throws IOException + { + // Retry transient NCBI failures (5xx, read timeouts) with exponential backoff; + // other failures (e.g. 4xx) are permanent and fail fast. + for (int attempt = 1; ; attempt++) + { + try + { + return executeGet(url); + } + catch (IOException e) + { + if (attempt >= MAX_HTTP_ATTEMPTS || !isRetryable(e)) + { + throw e; + } + long delayMs = retryDelayMs(attempt); + getLog(log).warn("NCBI request failed (attempt {} of {}); retrying in {} ms. URL: {}; cause: {}", + attempt, MAX_HTTP_ATTEMPTS, delayMs, url, e.toString()); + sleepMs(delayMs); + } + } + } + + private String executeGet(String url) throws IOException { ConnectionConfig connectionConfig = ConnectionConfig.custom() .setConnectTimeout(Timeout.ofMilliseconds(TIMEOUT_MS)) @@ -382,13 +415,69 @@ protected String getString(String url) throws IOException int status = response.getCode(); if (status < 200 || status >= 300) { - throw new HttpResponseException(status, response.getReasonPhrase()); + // Read the body only for client errors; 5xx bodies are typically large, + // uninformative HTML error pages. + String body = (status >= 400 && status < 500 && response.getEntity() != null) + ? EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8) + : null; + throw new HttpResponseException(status, errorDetail(status, response.getReasonPhrase(), body)); } return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); }); } } + /** + * Builds the message for a non-2xx HttpResponseException. For client errors (4xx) the response + * body is appended (e.g. NCBI's "API key invalid" message) so the cause is visible in the server + * log; for other statuses only the reason phrase is used (5xx bodies are uninformative HTML). + */ + private static String errorDetail(int status, String reasonPhrase, @Nullable String body) + { + if (status >= 400 && status < 500 && !StringUtils.isBlank(body)) + { + return reasonPhrase + " - " + StringUtils.abbreviate(body.strip(), 500); + } + return reasonPhrase; + } + + /** + * Transient NCBI failures worth retrying: read timeouts and 5xx responses. + * 4xx and other errors are treated as permanent and fail fast. + */ + private static boolean isRetryable(IOException e) + { + if (e instanceof SocketTimeoutException) + { + return true; + } + if (e instanceof HttpResponseException hre) + { + return hre.getStatusCode() >= 500; + } + return false; + } + + // Exponential backoff: 500ms after the 1st failure, 1000ms after the 2nd, etc. No jitter is + // needed because both callers (the daily reminder job and the UI search) issue NCBI requests + // sequentially, so a retry never collides with a sibling request from the same caller. + private static long retryDelayMs(int attempt) + { + return (long) RETRY_BASE_DELAY_MS << (attempt - 1); + } + + private static void sleepMs(long ms) + { + try + { + Thread.sleep(ms); + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + } + } + /** * Fetch metadata for PMC articles using ESummary API */ @@ -420,7 +509,7 @@ private Map fetchMetadata(Collection ids, String dat try { - JSONObject json = getJson(url); + JSONObject json = getJson(url, log); JSONObject result = json.optJSONObject("result"); if (result == null) return Collections.emptyMap(); @@ -921,14 +1010,7 @@ static List extractTitleKeywords(String title) */ private static void rateLimit() { - try - { - Thread.sleep(RATE_LIMIT_DELAY_MS); - } - catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } + sleepMs(RATE_LIMIT_DELAY_MS); } /** @@ -947,9 +1029,24 @@ static String stripQuerySpecialChars(String value) */ private static String buildCommonParams(String database) { - return "db=" + URLEncoder.encode(database, StandardCharsets.UTF_8) + + return buildCommonParams(database, PrivateDataReminderSettings.get().getNcbiApiKey()); + } + + // Builds the shared eutils query parameters. The API key is passed in (rather than looked up) + // so this can be unit tested without a running server. + private static String buildCommonParams(String database, @Nullable String apiKey) + { + String params = "db=" + URLEncoder.encode(database, StandardCharsets.UTF_8) + "&tool=" + URLEncoder.encode(TOOL, StandardCharsets.UTF_8) + "&email=" + URLEncoder.encode(EMAIL, StandardCharsets.UTF_8); + + // An NCBI API key (configured in the Private Data Reminder Settings) raises the eutils + // rate limit from 3 to 10 requests/sec. Append it when one has been entered. + if (!StringUtils.isBlank(apiKey)) + { + params += "&api_key=" + URLEncoder.encode(apiKey.trim(), StandardCharsets.UTF_8); + } + return params; } /** @@ -1422,6 +1519,66 @@ public void testPublicationMatchRoundTrip() assertFalse(restored.matchesProteomeXchangeId()); } + // -- HTTP retry tests -- + + @Test + public void testIsRetryable() + { + // Read timeouts and 5xx responses are transient NCBI failures -> retry + assertTrue(isRetryable(new SocketTimeoutException("Read timed out"))); + assertTrue(isRetryable(new HttpResponseException(500, "Internal Server Error"))); + assertTrue(isRetryable(new HttpResponseException(503, "Service Unavailable"))); + + // 4xx and generic IO errors are permanent -> fail fast + assertFalse(isRetryable(new HttpResponseException(400, "Bad Request"))); + assertFalse(isRetryable(new HttpResponseException(404, "Not Found"))); + assertFalse(isRetryable(new IOException("connection reset"))); + } + + @Test + public void testRetryDelayMs() + { + // Exponential backoff: 500ms, 1000ms, 2000ms per attempt. + assertEquals(500, retryDelayMs(1)); + assertEquals(1000, retryDelayMs(2)); + assertEquals(2000, retryDelayMs(3)); + } + + @Test + public void testErrorDetail() + { + // 4xx: the response body is appended so the cause (e.g. an invalid API key) is logged + String detail = errorDetail(400, "Bad Request", "{\"error\":\"API key invalid\"}"); + assertTrue(detail.contains("Bad Request")); + assertTrue(detail.contains("API key invalid")); + + // 5xx: body omitted (uninformative) + assertEquals("Internal Server Error", errorDetail(500, "Internal Server Error", "oops")); + + // 4xx with blank or null body: just the reason phrase, no trailing separator + assertEquals("Bad Request", errorDetail(400, "Bad Request", "")); + assertEquals("Bad Request", errorDetail(400, "Bad Request", null)); + } + + @Test + public void testBuildCommonParams() + { + // Always includes db, tool, email + String params = buildCommonParams("pmc", null); + assertTrue(params.contains("db=pmc")); + assertTrue(params.contains("tool=" + TOOL)); + assertTrue(params.contains("email=")); + + // No api_key when the key is null, empty, or blank + assertFalse("api_key should be absent when no key is set", params.contains("api_key")); + assertFalse(buildCommonParams("pmc", "").contains("api_key")); + assertFalse(buildCommonParams("pmc", " ").contains("api_key")); + + // api_key appended (and trimmed) when a key is set + assertTrue(buildCommonParams("pubmed", "ABC123").contains("api_key=ABC123")); + assertTrue(buildCommonParams("pmc", " ABC123 ").contains("api_key=ABC123")); + } + // -- Helper methods for building test JSON -- private static JSONObject articleMetadata(String source, String fullJournalName) diff --git a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp index cf5a738c..95f311fa 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp +++ b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp @@ -170,6 +170,19 @@ + + + <%=h(PrivateDataReminderSettings.PROP_NCBI_API_KEY)%> + + + +
+ Optional. An NCBI API key raises the request rate limit for PubMed/PMC searches from 3 to 10 per second. +
+ Create one under Account settings at ncbi.nlm.nih.gov. Leave blank to search without a key. +
+ + <%=button("Save").submit(true)%> <%=button("Cancel").href(panoramaPublicAdminUrl)%> diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index c4ee6f4c..c83fa289 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -631,7 +631,7 @@ protected void verifyIsPublicColumn(String panoramaPublicProject, String experim /** * Navigate to the Private Data Reminder Settings page and read the current form values. - * Returns a map with keys: extensionLength, delayUntilFirstReminder, reminderFrequency, enablePublicationSearch, publicationSearchFrequency. + * Returns a map with keys: extensionLength, delayUntilFirstReminder, reminderFrequency, enablePublicationSearch, publicationSearchFrequency, ncbiApiKey. */ protected Map getPrivateDataReminderSettings() { @@ -645,6 +645,7 @@ protected Map getPrivateDataReminderSettings() settings.put("reminderFrequency", getFormElement(Locator.input("reminderFrequency"))); settings.put("enablePublicationSearch", String.valueOf(Locator.checkboxByName("enablePublicationSearch").findElement(getDriver()).isSelected())); settings.put("publicationSearchFrequency", getFormElement(Locator.input("publicationSearchFrequency"))); + settings.put("ncbiApiKey", getFormElement(Locator.input("ncbiApiKey"))); return settings; } @@ -654,6 +655,14 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de } protected void savePrivateDataReminderSettings(String extensionLength, String delayUntilFirstReminder, String reminderFrequency, boolean enablePublicationSearch) + { + savePrivateDataReminderSettings(extensionLength, delayUntilFirstReminder, reminderFrequency, enablePublicationSearch, null); + } + + /** + * @param ncbiApiKey value to enter in the NCBI API key field; pass null to leave the field untouched. + */ + protected void savePrivateDataReminderSettings(String extensionLength, String delayUntilFirstReminder, String reminderFrequency, boolean enablePublicationSearch, String ncbiApiKey) { goToAdminConsole().goToSettingsSection(); clickAndWait(Locator.linkWithText("Panorama Public")); @@ -662,6 +671,10 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de setFormElement(Locator.input("delayUntilFirstReminder"), delayUntilFirstReminder); setFormElement(Locator.input("reminderFrequency"), reminderFrequency); setFormElement(Locator.input("extensionLength"), extensionLength); + if (ncbiApiKey != null) + { + setFormElement(Locator.input("ncbiApiKey"), ncbiApiKey); + } if (enablePublicationSearch) { checkCheckbox(Locator.checkboxByName("enablePublicationSearch")); @@ -677,6 +690,10 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de assertEquals(String.valueOf(delayUntilFirstReminder), getFormElement(Locator.input("delayUntilFirstReminder"))); assertEquals(String.valueOf(reminderFrequency), getFormElement(Locator.input("reminderFrequency"))); assertEquals(String.valueOf(extensionLength), getFormElement(Locator.input("extensionLength"))); + if (ncbiApiKey != null) + { + assertEquals(ncbiApiKey, getFormElement(Locator.input("ncbiApiKey"))); + } } protected void goToSendRemindersPage(String projectName) diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java index 555a193e..cb3e7f7f 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java @@ -114,6 +114,15 @@ public void testPublicationSearchAndDismiss() // Step 1: Set up mock NCBI service if running on TeamCity setupMockNcbiService(); + // Capture the existing reminder settings up front so doCleanup can restore them exactly. + // The dev machine may already have a real NCBI API key (and other non-default values) set. + _originalReminderSettings = getPrivateDataReminderSettings(); + + // Baseline server error count up front. The only errors this test should produce are the + // deliberate ones from the bad-key search at the end; checkExpectedErrors(baseline + n) + // verifies exactly that count and fails on any unexpected extras (rather than masking them). + int serverErrorCount = getServerErrorCount(); + // Step 2: Create dataset 1 folder, submit to Panorama Public, and copy String testProject = getProjectName(); String shortAccessUrl1 = setupFolderSubmitAndCopy(testProject, FOLDER_1, TARGET_FOLDER_1, @@ -190,8 +199,6 @@ public void testPublicationSearchAndDismiss() assertTextPresent("The user has already dismissed the publication suggestion PubMed ID " + PMID_1 + " for this dataset"); // Step 9: Run reminders in TEST MODE — verify DatasetStatus is NOT updated - // Save current settings so they can be restored in doCleanup - _originalReminderSettings = getPrivateDataReminderSettings(); savePrivateDataReminderSettings("2", "0", "0", true); // Post reminders in test mode with publication search enabled @@ -256,6 +263,32 @@ public void testPublicationSearchAndDismiss() assertNotNull("Expected publicationType for dataset 2", dsStatus2AfterPost.get("PublicationType")); assertNotNull("Expected lastReminderDate for dataset 2", dsStatus2AfterPost.get("LastReminderDate")); assertNotNull("Expected citation to be cached for dataset 2", dsStatus2AfterPost.get("Citation")); + + // Verify the NCBI API key setting round-trips (set -> save -> re-read). The helper asserts + // the saved value is reflected on the form. doCleanup restores the original settings. + savePrivateDataReminderSettings("2", "0", "0", true, "test-ncbi-api-key"); + + // Verify the configured key actually reaches the live eutils requests. NCBI rejects an + // invalid key with HTTP 400, so re-searching dataset 2 (which found a publication above) + // should now find nothing. Only meaningful against the real NCBI service: on TeamCity the + // mock service bypasses the key, so this runs only when not using the mock (i.e. on dev). + if (!_useMockNcbi) + { + searchPublicationsForDataset(panoramaPublicProject, TARGET_FOLDER_2, exptId2); + assertTextPresent("No publications found for this dataset."); + assertTextNotPresent(PMID_2); + + // The invalid key makes NCBI return HTTP 400. Verify the server log records the cause: + // this proves both that the key reached eutils and that the 400 response body is logged. + assertTrue("Server log should record NCBI's invalid-key error", + getServerErrors().contains("API key invalid")); + + // The bad-key search logs one error per failed NCBI call: 2 PMC strategy searches for + // dataset 2 (ProteomeXchange ID and Panorama URL) plus the PubMed fallback = 3. + // checkExpectedErrors verifies exactly these and clears them; unlike a bare resetErrors() + // it fails the test if any other unexpected errors occurred during the run. + checkExpectedErrors(serverErrorCount + 3); + } } /* @@ -486,7 +519,8 @@ public void resetAfterTest() _originalReminderSettings.get("extensionLength"), _originalReminderSettings.get("delayUntilFirstReminder"), _originalReminderSettings.get("reminderFrequency"), - Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch"))); + Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch")), + _originalReminderSettings.get("ncbiApiKey")); } } From 02022c47be0d2db4944fb4fa93b15954a02503b9 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sat, 13 Jun 2026 21:22:35 -0700 Subject: [PATCH 02/24] Captured PublicationSearchTest's server error-count baseline just before the bad-key search so incidental NCBI 5xx in earlier steps don't make checkExpectedErrors flaky on dev --- .../tests/panoramapublic/PublicationSearchTest.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java index cb3e7f7f..c76e0895 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java @@ -118,11 +118,6 @@ public void testPublicationSearchAndDismiss() // The dev machine may already have a real NCBI API key (and other non-default values) set. _originalReminderSettings = getPrivateDataReminderSettings(); - // Baseline server error count up front. The only errors this test should produce are the - // deliberate ones from the bad-key search at the end; checkExpectedErrors(baseline + n) - // verifies exactly that count and fails on any unexpected extras (rather than masking them). - int serverErrorCount = getServerErrorCount(); - // Step 2: Create dataset 1 folder, submit to Panorama Public, and copy String testProject = getProjectName(); String shortAccessUrl1 = setupFolderSubmitAndCopy(testProject, FOLDER_1, TARGET_FOLDER_1, @@ -274,6 +269,12 @@ public void testPublicationSearchAndDismiss() // mock service bypasses the key, so this runs only when not using the mock (i.e. on dev). if (!_useMockNcbi) { + // Capture the server error count immediately before the deliberate bad-key search so the + // assertion below counts only its errors. Capturing at test start would also count any + // incidental transient NCBI errors (5xx) from the earlier real-NCBI steps, which the + // retry logic reduces but cannot eliminate, making the check flaky on a dev machine. + int serverErrorCount = getServerErrorCount(); + searchPublicationsForDataset(panoramaPublicProject, TARGET_FOLDER_2, exptId2); assertTextPresent("No publications found for this dataset."); assertTextNotPresent(PMID_2); From 21a0cc33824c6c79f9afe3a23b14c43c152d8c4f Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sat, 13 Jun 2026 21:31:57 -0700 Subject: [PATCH 03/24] Asserted enablePublicationSearch round-trips through save in PanoramaPublicBaseTest's reminder-settings helper --- .../test/tests/panoramapublic/PanoramaPublicBaseTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index c83fa289..1483ba93 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -690,6 +690,8 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de assertEquals(String.valueOf(delayUntilFirstReminder), getFormElement(Locator.input("delayUntilFirstReminder"))); assertEquals(String.valueOf(reminderFrequency), getFormElement(Locator.input("reminderFrequency"))); assertEquals(String.valueOf(extensionLength), getFormElement(Locator.input("extensionLength"))); + assertEquals("enablePublicationSearch should round-trip through save", enablePublicationSearch, + Locator.checkboxByName("enablePublicationSearch").findElement(getDriver()).isSelected()); if (ncbiApiKey != null) { assertEquals(ncbiApiKey, getFormElement(Locator.input("ncbiApiKey"))); From 3c4bdfd0c86a8fccf94f235be244cc8dfd2737ab Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sat, 13 Jun 2026 21:44:43 -0700 Subject: [PATCH 04/24] Scoped MockNcbiPublicationSearchService matching to the decoded db/term/id query params instead of substring-scanning the whole request URL --- .../MockNcbiPublicationSearchService.java | 63 +++++++++++++------ 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java index b9995737..6f0df488 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java @@ -21,7 +21,10 @@ import org.json.JSONObject; import java.io.IOException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -152,15 +155,24 @@ else if (url.contains("lit/ctxp")) private JSONObject handleESearch(String url) { - boolean isPmc = url.contains("db=pmc"); + boolean isPmc = "pmc".equals(extractQueryParam(url, "db")); Map> searchMap = isPmc ? _pmcSearchResults : _pubmedSearchResults; + // Match registered search keys against the decoded ESearch query term only, not the whole + // URL, so a key cannot accidentally match part of another parameter (tool/email) or another + // key. The real ESearch term wraps the key in quotes (e.g. "PXD056793"), so contains() on + // the term is the right granularity. + String term = extractQueryParam(url, "term"); + JSONArray idList = new JSONArray(); - for (Map.Entry> entry : searchMap.entrySet()) + if (term != null) { - if (url.contains(entry.getKey())) + for (Map.Entry> entry : searchMap.entrySet()) { - entry.getValue().forEach(idList::put); + if (term.contains(entry.getKey())) + { + entry.getValue().forEach(idList::put); + } } } @@ -171,13 +183,18 @@ private JSONObject handleESearch(String url) private JSONObject handleESummary(String url) { - boolean isPmc = url.contains("db=pmc"); + boolean isPmc = "pmc".equals(extractQueryParam(url, "db")); Map metadataMap = isPmc ? _pmcMetadata : _pubmedMetadata; + // ESummary requests a comma-separated list of IDs in the "id" parameter. Match registered + // IDs against that list (exactly, not by substring), rather than scanning the whole URL. + String idParam = extractQueryParam(url, "id"); + List requestedIds = idParam == null ? List.of() : Arrays.asList(idParam.split(",")); + JSONObject result = new JSONObject(); for (Map.Entry entry : metadataMap.entrySet()) { - if (url.contains(entry.getKey())) + if (requestedIds.contains(entry.getKey())) { result.put(entry.getKey(), entry.getValue()); } @@ -193,20 +210,7 @@ private JSONObject handleESummary(String url) */ private JSONObject handleCitation(String url) { - // Extract the publication ID from the URL (last segment after "id=") - String id = null; - int idIdx = url.indexOf("id="); - if (idIdx >= 0) - { - id = url.substring(idIdx + 3); - // Remove any trailing query parameters - int ampIdx = id.indexOf('&'); - if (ampIdx >= 0) - { - id = id.substring(0, ampIdx); - } - } - + String id = extractQueryParam(url, "id"); String citation = id != null ? _citations.get(id) : null; if (citation != null) { @@ -214,4 +218,23 @@ private JSONObject handleCitation(String url) } return new JSONObject(); } + + /** + * Returns the URL-decoded value of the given query parameter, or null if it is not present. + * Used to scope mock matching to a specific parameter (db, term, id) instead of the whole URL. + */ + private static @Nullable String extractQueryParam(String url, String name) + { + int queryStart = url.indexOf('?'); + String query = queryStart >= 0 ? url.substring(queryStart + 1) : url; + for (String pair : query.split("&")) + { + int eq = pair.indexOf('='); + if (eq > 0 && pair.substring(0, eq).equals(name)) + { + return URLDecoder.decode(pair.substring(eq + 1), StandardCharsets.UTF_8); + } + } + return null; + } } From 0c2058fcdc65f9ae09528d707e40dc9344a4d2a3 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sat, 13 Jun 2026 22:03:35 -0700 Subject: [PATCH 05/24] Added unit tests for the getString retry loop (retry-then-succeed, give-up-after-MAX_HTTP_ATTEMPTS, no-retry-on-4xx); made executeGet protected so a test subclass can drive it - Corrected stale comments in NcbiPublicationSearchServiceImpl and its mock, and tightened the others. --- .../MockNcbiPublicationSearchService.java | 13 +- .../NcbiPublicationSearchServiceImpl.java | 122 ++++++++++++++---- .../PanoramaPublicBaseTest.java | 7 +- .../panoramapublic/PublicationSearchTest.java | 25 ++-- 4 files changed, 120 insertions(+), 47 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java index 6f0df488..067dabed 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java @@ -33,7 +33,8 @@ * Mock implementation of {@link NcbiPublicationSearchService} that returns canned data registered by tests. * Used by Selenium tests when running on TeamCity. * Extends {@link NcbiPublicationSearchServiceImpl} and only overrides {@link #getString(String, Logger)}, - * the single method that makes HTTP calls to NCBI. All search logic, filtering, author/title + * the method every NCBI request passes through. The override takes the place of the real HTTP request + * in {@code executeGet()} and the retry loop around it. All search logic, filtering, author/title * verification, citation parsing, and priority filtering run through the real implementation code. * Tests register mock articles via {@link #register}, providing the database, ID, search key, * metadata fields, and citation. The mock builds internal lookup maps from this data and returns @@ -54,7 +55,7 @@ public class MockNcbiPublicationSearchService extends NcbiPublicationSearchServi /** * Register a mock article. The mock stores the data in internal lookup maps used by - * {@link #getString(String)}. + * {@link #getString(String, Logger)}. * @param database "pmc" or "pubmed" — the NCBI database this article is in * @param id the article ID in the given database (numeric ID for pmc or pubmed) * @param searchKey what ESearch query term finds this article (e.g. PXD ID for PMC, author last name for PubMed) @@ -158,10 +159,9 @@ private JSONObject handleESearch(String url) boolean isPmc = "pmc".equals(extractQueryParam(url, "db")); Map> searchMap = isPmc ? _pmcSearchResults : _pubmedSearchResults; - // Match registered search keys against the decoded ESearch query term only, not the whole - // URL, so a key cannot accidentally match part of another parameter (tool/email) or another - // key. The real ESearch term wraps the key in quotes (e.g. "PXD056793"), so contains() on - // the term is the right granularity. + // Match registered search keys against the decoded ESearch query term, so a key cannot + // match part of another parameter such as tool or email. The real ESearch term wraps the + // key in quotes (e.g. "PXD056793"), so contains() on the term is the right granularity. String term = extractQueryParam(url, "term"); JSONArray idList = new JSONArray(); @@ -221,7 +221,6 @@ private JSONObject handleCitation(String url) /** * Returns the URL-decoded value of the given query parameter, or null if it is not present. - * Used to scope mock matching to a specific parameter (db, term, id) instead of the whole URL. */ private static @Nullable String extractQueryParam(String url, String name) { diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index 3f6fd727..908e442e 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -94,7 +94,7 @@ public static void setInstance(NcbiPublicationSearchService impl) private static final String PMC_CITATION_EXPORTER_URL = "https://api.ncbi.nlm.nih.gov/lit/ctxp/v1/pmc/?format=citation&id="; // API parameters - private static final int RATE_LIMIT_DELAY_MS = 400; // NCBI allows 3 requests/sec + private static final int RATE_LIMIT_DELAY_MS = 400; // NCBI allows 3 requests/sec without an API key private static final int TIMEOUT_MS = 10000; // 10 seconds // NCBI eutils intermittently returns transient 5xx errors and read timeouts, even well under @@ -363,14 +363,12 @@ protected JSONObject getJson(String url, Logger log) throws IOException /** * Execute an HTTP GET request and return the response body as a string. Retry warnings are - * written to {@code log} so they land in the pipeline job log when invoked from the reminder - * job (and in the server log for UI-triggered searches, which pass the static logger). + * written to {@code log}. The reminder job passes its pipeline job logger, and UI-triggered + * searches pass the static server logger. * @throws IOException if the request fails or the server returns a non-2xx response */ protected String getString(String url, Logger log) throws IOException { - // Retry transient NCBI failures (5xx, read timeouts) with exponential backoff; - // other failures (e.g. 4xx) are permanent and fail fast. for (int attempt = 1; ; attempt++) { try @@ -384,14 +382,15 @@ protected String getString(String url, Logger log) throws IOException throw e; } long delayMs = retryDelayMs(attempt); - getLog(log).warn("NCBI request failed (attempt {} of {}); retrying in {} ms. URL: {}; cause: {}", + getLog(log).warn("NCBI request failed (attempt {} of {}). Retrying in {} ms. URL: {}. Cause: {}", attempt, MAX_HTTP_ATTEMPTS, delayMs, url, e.toString()); sleepMs(delayMs); } } } - private String executeGet(String url) throws IOException + // Overridden in unit tests to drive the retry loop in getString() without real HTTP calls. + protected String executeGet(String url) throws IOException { ConnectionConfig connectionConfig = ConnectionConfig.custom() .setConnectTimeout(Timeout.ofMilliseconds(TIMEOUT_MS)) @@ -415,8 +414,8 @@ private String executeGet(String url) throws IOException int status = response.getCode(); if (status < 200 || status >= 300) { - // Read the body only for client errors; 5xx bodies are typically large, - // uninformative HTML error pages. + // 5xx bodies are large, uninformative HTML error pages, so only client error + // bodies are read. String body = (status >= 400 && status < 500 && response.getEntity() != null) ? EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8) : null; @@ -428,9 +427,9 @@ private String executeGet(String url) throws IOException } /** - * Builds the message for a non-2xx HttpResponseException. For client errors (4xx) the response - * body is appended (e.g. NCBI's "API key invalid" message) so the cause is visible in the server - * log; for other statuses only the reason phrase is used (5xx bodies are uninformative HTML). + * Build the message for a non-2xx HttpResponseException. For client errors (4xx) the response + * body is appended, so NCBI's reason (e.g. "API key invalid") reaches the log. Other statuses + * use only the reason phrase. */ private static String errorDetail(int status, String reasonPhrase, @Nullable String body) { @@ -442,8 +441,8 @@ private static String errorDetail(int status, String reasonPhrase, @Nullable Str } /** - * Transient NCBI failures worth retrying: read timeouts and 5xx responses. - * 4xx and other errors are treated as permanent and fail fast. + * Read timeouts and 5xx responses are transient NCBI failures worth retrying. 4xx and other + * errors are permanent. */ private static boolean isRetryable(IOException e) { @@ -458,9 +457,9 @@ private static boolean isRetryable(IOException e) return false; } - // Exponential backoff: 500ms after the 1st failure, 1000ms after the 2nd, etc. No jitter is - // needed because both callers (the daily reminder job and the UI search) issue NCBI requests - // sequentially, so a retry never collides with a sibling request from the same caller. + // 500ms after the first failure, then doubling. Jitter is not needed. Both callers, the daily + // reminder job and the UI search, issue NCBI requests sequentially, so a retry never collides + // with a sibling request. private static long retryDelayMs(int attempt) { return (long) RETRY_BASE_DELAY_MS << (attempt - 1); @@ -1025,15 +1024,15 @@ static String stripQuerySpecialChars(String value) } /** - * Build the common NCBI API parameters (db, tool, email) with URL encoding. + * Build the common NCBI API parameters (db, tool, email, and api_key when one is configured) + * with URL encoding. */ private static String buildCommonParams(String database) { return buildCommonParams(database, PrivateDataReminderSettings.get().getNcbiApiKey()); } - // Builds the shared eutils query parameters. The API key is passed in (rather than looked up) - // so this can be unit tested without a running server. + // The API key is passed in rather than read from the settings, so this overload runs without a server. private static String buildCommonParams(String database, @Nullable String apiKey) { String params = "db=" + URLEncoder.encode(database, StandardCharsets.UTF_8) + @@ -1041,7 +1040,7 @@ private static String buildCommonParams(String database, @Nullable String apiKey "&email=" + URLEncoder.encode(EMAIL, StandardCharsets.UTF_8); // An NCBI API key (configured in the Private Data Reminder Settings) raises the eutils - // rate limit from 3 to 10 requests/sec. Append it when one has been entered. + // rate limit from 3 to 10 requests/sec. if (!StringUtils.isBlank(apiKey)) { params += "&api_key=" + URLEncoder.encode(apiKey.trim(), StandardCharsets.UTF_8); @@ -1538,7 +1537,7 @@ public void testIsRetryable() @Test public void testRetryDelayMs() { - // Exponential backoff: 500ms, 1000ms, 2000ms per attempt. + // Exponential backoff of 500ms, 1000ms, 2000ms per attempt. assertEquals(500, retryDelayMs(1)); assertEquals(1000, retryDelayMs(2)); assertEquals(2000, retryDelayMs(3)); @@ -1579,6 +1578,85 @@ public void testBuildCommonParams() assertTrue(buildCommonParams("pmc", " ABC123 ").contains("api_key=ABC123")); } + @Test + public void testGetStringRetriesTransientFailures() throws IOException + { + // executeGet returns a 5xx twice, then succeeds. getString should retry and return the body. + int[] attempts = {0}; + NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() + { + @Override + protected String executeGet(String url) throws IOException + { + if (++attempts[0] < 3) + throw new HttpResponseException(503, "Service Unavailable"); + return "body"; + } + }; + assertEquals("body", service.getString("http://test", LOG)); + assertEquals("Should retry until the 3rd attempt succeeds", 3, attempts[0]); + } + + @Test + public void testGetStringGivesUpAfterMaxAttempts() + { + // executeGet always returns a 5xx. getString should try MAX_HTTP_ATTEMPTS times, then rethrow. + int[] attempts = {0}; + NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() + { + @Override + protected String executeGet(String url) throws IOException + { + attempts[0]++; + throw new HttpResponseException(500, "Internal Server Error"); + } + }; + try + { + service.getString("http://test", LOG); + fail("Expected HttpResponseException after exhausting retries"); + } + catch (HttpResponseException e) + { + assertEquals(500, e.getStatusCode()); + } + catch (IOException e) + { + fail("Expected HttpResponseException, got " + e); + } + assertEquals("Should attempt exactly MAX_HTTP_ATTEMPTS times", MAX_HTTP_ATTEMPTS, attempts[0]); + } + + @Test + public void testGetStringDoesNotRetryClientErrors() + { + // A 4xx is permanent. getString should fail immediately without retrying. + int[] attempts = {0}; + NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() + { + @Override + protected String executeGet(String url) throws IOException + { + attempts[0]++; + throw new HttpResponseException(400, "Bad Request"); + } + }; + try + { + service.getString("http://test", LOG); + fail("Expected HttpResponseException for a 4xx"); + } + catch (HttpResponseException e) + { + assertEquals(400, e.getStatusCode()); + } + catch (IOException e) + { + fail("Expected HttpResponseException, got " + e); + } + assertEquals("4xx must not be retried", 1, attempts[0]); + } + // -- Helper methods for building test JSON -- private static JSONObject articleMetadata(String source, String fullJournalName) diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index 1483ba93..3637108c 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -660,7 +660,7 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de } /** - * @param ncbiApiKey value to enter in the NCBI API key field; pass null to leave the field untouched. + * @param ncbiApiKey value to enter in the NCBI API key field. Pass null to leave the field untouched. */ protected void savePrivateDataReminderSettings(String extensionLength, String delayUntilFirstReminder, String reminderFrequency, boolean enablePublicationSearch, String ncbiApiKey) { @@ -690,11 +690,12 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de assertEquals(String.valueOf(delayUntilFirstReminder), getFormElement(Locator.input("delayUntilFirstReminder"))); assertEquals(String.valueOf(reminderFrequency), getFormElement(Locator.input("reminderFrequency"))); assertEquals(String.valueOf(extensionLength), getFormElement(Locator.input("extensionLength"))); - assertEquals("enablePublicationSearch should round-trip through save", enablePublicationSearch, + assertEquals("The saved publication search setting should be displayed on the form", enablePublicationSearch, Locator.checkboxByName("enablePublicationSearch").findElement(getDriver()).isSelected()); if (ncbiApiKey != null) { - assertEquals(ncbiApiKey, getFormElement(Locator.input("ncbiApiKey"))); + assertEquals("The saved NCBI API key should be displayed on the form", ncbiApiKey, + getFormElement(Locator.input("ncbiApiKey"))); } } diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java index c76e0895..898f64b7 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java @@ -259,35 +259,30 @@ public void testPublicationSearchAndDismiss() assertNotNull("Expected lastReminderDate for dataset 2", dsStatus2AfterPost.get("LastReminderDate")); assertNotNull("Expected citation to be cached for dataset 2", dsStatus2AfterPost.get("Citation")); - // Verify the NCBI API key setting round-trips (set -> save -> re-read). The helper asserts - // the saved value is reflected on the form. doCleanup restores the original settings. + // Verify the NCBI API key setting round-trips (set -> save -> re-read). savePrivateDataReminderSettings("2", "0", "0", true, "test-ncbi-api-key"); - // Verify the configured key actually reaches the live eutils requests. NCBI rejects an - // invalid key with HTTP 400, so re-searching dataset 2 (which found a publication above) - // should now find nothing. Only meaningful against the real NCBI service: on TeamCity the - // mock service bypasses the key, so this runs only when not using the mock (i.e. on dev). + // Verify the configured key reaches the live eutils requests. NCBI rejects an invalid key + // with HTTP 400, so re-searching dataset 2, which found a publication above, should now + // find nothing. The mock service bypasses the key, so this runs only against real NCBI. if (!_useMockNcbi) { // Capture the server error count immediately before the deliberate bad-key search so the - // assertion below counts only its errors. Capturing at test start would also count any - // incidental transient NCBI errors (5xx) from the earlier real-NCBI steps, which the - // retry logic reduces but cannot eliminate, making the check flaky on a dev machine. + // assertion below only counts errors due to the bad-key search. int serverErrorCount = getServerErrorCount(); searchPublicationsForDataset(panoramaPublicProject, TARGET_FOLDER_2, exptId2); assertTextPresent("No publications found for this dataset."); assertTextNotPresent(PMID_2); - // The invalid key makes NCBI return HTTP 400. Verify the server log records the cause: - // this proves both that the key reached eutils and that the 400 response body is logged. + // The invalid key makes NCBI return HTTP 400, and errorDetail appends the response body + // to the logged message. assertTrue("Server log should record NCBI's invalid-key error", getServerErrors().contains("API key invalid")); - // The bad-key search logs one error per failed NCBI call: 2 PMC strategy searches for - // dataset 2 (ProteomeXchange ID and Panorama URL) plus the PubMed fallback = 3. - // checkExpectedErrors verifies exactly these and clears them; unlike a bare resetErrors() - // it fails the test if any other unexpected errors occurred during the run. + // The bad-key search logs one error per failed NCBI call. Dataset 2 runs two PMC + // strategy searches, on ProteomeXchange ID and Panorama URL, plus the PubMed fallback. + // checkExpectedErrors clears exactly that many and fails on any others. checkExpectedErrors(serverErrorCount + 3); } } From 74370ceafc1da52eb68f9a1aa8ab556c54e0cf47 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Mon, 31 Aug 2026 13:46:17 -0700 Subject: [PATCH 06/24] Protected the NCBI API key and made a failed publication search visible * Redacted the API key from the retry warning and from NCBI's 4xx body before either reaches a log * Moved the key to the encrypted property store, and stopped the form displaying or erasing it * Threw NcbiSearchException from executeSearch so a rejected key no longer reads as a dataset with no paper * Added a Validate button that checks a key against NCBI before it is saved * Scheduled the reminder job from startBackgroundThreads so a SchedulerException cannot fail server startup * Added redaction unit tests and reworked the Selenium NCBI API key coverage Co-Authored-By: Claude --- .../PanoramaPublicController.java | 83 +++++++++++++- .../panoramapublic/PanoramaPublicModule.java | 21 +++- .../message/PrivateDataReminderSettings.java | 40 ++++++- .../ncbi/NcbiPublicationSearchService.java | 14 +++ .../NcbiPublicationSearchServiceImpl.java | 101 ++++++++++++++++-- .../ncbi/NcbiSearchException.java | 34 ++++++ .../pipeline/PrivateDataReminderJob.java | 38 ++++++- .../view/privateDataRemindersSettingsForm.jsp | 62 ++++++++++- .../PanoramaPublicBaseTest.java | 8 +- .../panoramapublic/PublicationSearchTest.java | 58 ++++++---- 10 files changed, 413 insertions(+), 46 deletions(-) create mode 100644 panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java index a9dc4619..c3d0ed24 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java @@ -10084,6 +10084,52 @@ public static ActionURL getViewExperimentModificationsURL(int experimentAnnotati return result; } + @RequiresPermission(AdminOperationsPermission.class) + public static class ValidateNcbiApiKeyAction extends MutatingApiAction + { + @Override + public Object execute(PrivateDataReminderSettingsForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("success", true); + + // An empty field means check the key that is already saved, since the form never + // displays it. + boolean checkingSavedKey = StringUtils.isBlank(form.getNcbiApiKey()); + String apiKey = checkingSavedKey + ? PrivateDataReminderSettings.get().getNcbiApiKey() + : form.getNcbiApiKey().trim(); + + if (StringUtils.isBlank(apiKey)) + { + response.put("valid", false); + response.put("message", "Enter a key to validate, or save one first."); + return response; + } + + String error = NcbiPublicationSearchService.get().validateApiKey(apiKey); + response.put("valid", error == null); + if (error == null) + { + // Validating does not store anything, so say so. Otherwise "accepted" reads as + // confirmation that the key is now in effect. + response.put("message", checkingSavedKey + ? "NCBI accepted the saved key." + : "NCBI accepted this key. Click Save to store it."); + LOG.info("NCBI accepted an API key entered on the Private Data Reminder Settings page."); + } + else + { + // The short message goes beside the field. NCBI's own words are offered separately, + // since the admin holding the key is the one who has to act on them. + response.put("message", "NCBI rejected this key."); + response.put("detail", error); + LOG.warn("NCBI rejected an API key entered on the Private Data Reminder Settings page. {}", error); + } + return response; + } + } + @RequiresPermission(AdminOperationsPermission.class) public static class PrivateDataReminderSettingsAction extends FormViewAction { @@ -10146,7 +10192,8 @@ public ModelAndView getView(PrivateDataReminderSettingsForm form, boolean reshow form.setExtensionLength(settings.getExtensionLength()); form.setEnablePublicationSearch(settings.isEnablePublicationSearch()); form.setPublicationSearchFrequency(settings.getPublicationSearchFrequency()); - form.setNcbiApiKey(settings.getNcbiApiKey()); + // Do not put the saved key in the form. The JSP shows only whether one is stored. + form.setNcbiApiKeySet(PrivateDataReminderSettings.hasNcbiApiKey()); } VBox view = new VBox(); @@ -10167,9 +10214,19 @@ public boolean handlePost(PrivateDataReminderSettingsForm form, BindException er settings.setExtensionLength(form.getExtensionLength()); settings.setEnablePublicationSearch(form.isEnablePublicationSearch()); settings.setPublicationSearchFrequency(form.getPublicationSearchFrequency()); - settings.setNcbiApiKey(form.getNcbiApiKey()); PrivateDataReminderSettings.save(settings); + // A blank field leaves the saved key alone, so editing the reminder schedule cannot + // erase it. Removing a key takes the explicit checkbox. + if (form.isClearNcbiApiKey()) + { + PrivateDataReminderSettings.saveNcbiApiKey(null); + } + else if (!StringUtils.isBlank(form.getNcbiApiKey())) + { + PrivateDataReminderSettings.saveNcbiApiKey(form.getNcbiApiKey()); + } + PrivateDataMessageScheduler.getInstance().initialize(settings.isEnableReminders()); return true; } @@ -10208,6 +10265,8 @@ public static class PrivateDataReminderSettingsForm private boolean _enablePublicationSearch; private Integer _publicationSearchFrequency; private String _ncbiApiKey; + private boolean _clearNcbiApiKey; + private boolean _ncbiApiKeySet; public boolean isEnabled() { @@ -10288,6 +10347,26 @@ public void setNcbiApiKey(String ncbiApiKey) { _ncbiApiKey = ncbiApiKey; } + + public boolean isClearNcbiApiKey() + { + return _clearNcbiApiKey; + } + + public void setClearNcbiApiKey(boolean clearNcbiApiKey) + { + _clearNcbiApiKey = clearNcbiApiKey; + } + + public boolean isNcbiApiKeySet() + { + return _ncbiApiKeySet; + } + + public void setNcbiApiKeySet(boolean ncbiApiKeySet) + { + _ncbiApiKeySet = ncbiApiKeySet; + } } @RequiresPermission(AdminOperationsPermission.class) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java index c7df83fb..6942ac20 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java @@ -16,6 +16,7 @@ package org.labkey.panoramapublic; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.admin.FolderSerializationRegistry; @@ -34,6 +35,7 @@ import org.labkey.api.security.roles.RoleManager; import org.labkey.api.settings.AdminConsole; import org.labkey.api.targetedms.TargetedMSService; +import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.BaseWebPartFactory; import org.labkey.api.view.HtmlView; @@ -83,6 +85,8 @@ public class PanoramaPublicModule extends SpringModule { + private static final Logger LOG = LogHelper.getLogger(PanoramaPublicModule.class, "Panorama Public module"); + public static final String NAME = "PanoramaPublic"; public static final String DOWNLOAD_DATA_INFO_WP = "Download Data"; @@ -153,8 +157,21 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) fileContentService.addFileListener(new PanoramaPublicFileListener()); } - // Start the private data reminder job on server restart if it is enabled. - PrivateDataMessageScheduler.getInstance().initialize(PrivateDataReminderSettings.get().isEnableReminders()); + } + + @Override + public void startBackgroundThreads() + { + // Re-establish the reminder schedule on every startup. Reminder messages contain absolute + // URLs, which are only safe to build once this method is called. + try + { + PrivateDataMessageScheduler.getInstance().initialize(PrivateDataReminderSettings.get().isEnableReminders()); + } + catch (RuntimeException e) + { + LOG.error("Failed to schedule the Panorama Public private data reminder job", e); + } } @NotNull diff --git a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java index 42cfd43b..9453b70d 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java +++ b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java @@ -15,6 +15,7 @@ */ package org.labkey.panoramapublic.message; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.Assert; @@ -30,6 +31,7 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.Date; +import java.util.Map; public class PrivateDataReminderSettings { @@ -42,6 +44,7 @@ public class PrivateDataReminderSettings public static final String PROP_ENABLE_PUBLICATION_SEARCH = "Enable publication search"; public static final String PROP_PUBLICATION_SEARCH_FREQUENCY = "Publication search frequency (months)"; public static final String PROP_NCBI_API_KEY = "NCBI API key"; + public static final String PROP_NCBI_CREDENTIALS = "Panorama Public NCBI credentials"; private static final boolean DEFAULT_ENABLE_REMINDERS = false; public static final String DEFAULT_REMINDER_TIME = "8:00 AM"; @@ -104,7 +107,7 @@ public static PrivateDataReminderSettings get() : Integer.valueOf(settingsMap.get(PROP_PUBLICATION_SEARCH_FREQUENCY)); settings.setPublicationSearchFrequency(publicationSearchFrequency); - settings.setNcbiApiKey(settingsMap.get(PROP_NCBI_API_KEY)); + settings.setNcbiApiKey(getNcbiApiKeyValue()); } else { @@ -151,10 +154,43 @@ public static void save(PrivateDataReminderSettings settings) settingsMap.put(PROP_REMINDER_TIME, settings.getReminderTimeFormatted()); settingsMap.put(PROP_ENABLE_PUBLICATION_SEARCH, String.valueOf(settings.isEnablePublicationSearch())); settingsMap.put(PROP_PUBLICATION_SEARCH_FREQUENCY, String.valueOf(settings.getPublicationSearchFrequency())); - settingsMap.put(PROP_NCBI_API_KEY, settings.getNcbiApiKey() != null ? settings.getNcbiApiKey() : ""); + // The API key is a credential and is saved separately, in the encrypted store. Saving the + // rest of the settings must never change it. + settingsMap.remove(PROP_NCBI_API_KEY); settingsMap.save(); } + /** + * Save the NCBI API key, or remove it when the key is blank. The key lives in the encrypted + * store, like the other credentials this module holds. + */ + public static void saveNcbiApiKey(@Nullable String apiKey) + { + PropertyManager.WritablePropertyMap credentials = + PropertyManager.getEncryptedStore().getWritableProperties(PROP_NCBI_CREDENTIALS, true); + if (StringUtils.isBlank(apiKey)) + { + credentials.remove(PROP_NCBI_API_KEY); + } + else + { + credentials.put(PROP_NCBI_API_KEY, apiKey.trim()); + } + credentials.save(); + } + + public static boolean hasNcbiApiKey() + { + return !StringUtils.isBlank(getNcbiApiKeyValue()); + } + + private static @Nullable String getNcbiApiKeyValue() + { + Map credentials = + PropertyManager.getEncryptedStore().getProperties(PROP_NCBI_CREDENTIALS); + return credentials.get(PROP_NCBI_API_KEY); + } + public void setEnableReminders(boolean enableReminders) { _enableReminders = enableReminders; diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java index 0e3de30d..e0b05baf 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java @@ -38,13 +38,27 @@ static NcbiPublicationSearchService get() @Nullable String getCitation(String publicationId, DB database); + /** + * Send a minimal request to NCBI with the given key. + * @return null if NCBI accepted the key, otherwise the reason it gave. + */ + @Nullable String validateApiKey(@Nullable String apiKey); + @Nullable Pair getPubMedLinkAndCitation(String pubmedId); /** * Searches PMC and PubMed for a publication associated with the experiment. * Returns the top match (highest priority) if multiple matches are found, or null if none. */ + /** + * @throws NcbiSearchException if a request to NCBI fails. An empty result therefore means no + * publication was found, not that the search could not be run. + */ @Nullable PublicationMatch searchForPublication(@NotNull ExperimentAnnotations expAnnotations, @Nullable Logger logger); + /** + * @throws NcbiSearchException if a request to NCBI fails. An empty result therefore means no + * publication was found, not that the search could not be run. + */ List searchForPublication(@NotNull ExperimentAnnotations expAnnotations, int maxResults, @Nullable Logger logger, boolean getCitations); } diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index 908e442e..a7dae20a 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -59,6 +59,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.labkey.panoramapublic.ncbi.PublicationMatch.MATCH_DOI; @@ -102,6 +104,9 @@ public static void setInstance(NcbiPublicationSearchService impl) private static final int MAX_HTTP_ATTEMPTS = 3; // initial try + 2 retries private static final int RETRY_BASE_DELAY_MS = 500; // exponential backoff base + private static final String REDACTED = "REDACTED"; + private static final Pattern API_KEY_PARAM = Pattern.compile("api_key=([^&\\s]*)"); + // NCBI suggests using the 'tool' and 'email' parameters on E-utilities URLs // https://www.nlm.nih.gov/dataguide/eutilities/utilities.html private static final String TOOL = "PanoramaPublic"; @@ -347,7 +352,24 @@ private List executeSearch(String query, String database, Logger log) catch (IOException | JSONException e) { log.error("Error searching {} with query: {}", database, query, e); - return Collections.emptyList(); + throw new NcbiSearchException("Error searching " + database + " with query: " + query, e); + } + } + + @Override + public @Nullable String validateApiKey(@Nullable String apiKey) + { + // A minimal ESearch request. NCBI answers a rejected key with 400 and a reason, which the + // retry loop does not retry, so this returns quickly either way. + String url = ESEARCH_URL + "?" + buildCommonParams("pubmed", apiKey) + "&term=labkey&retmax=1&retmode=json"; + try + { + getString(url, LOG); + return null; + } + catch (IOException e) + { + return redactApiKey(e.getMessage(), apiKey); } } @@ -383,7 +405,7 @@ protected String getString(String url, Logger log) throws IOException } long delayMs = retryDelayMs(attempt); getLog(log).warn("NCBI request failed (attempt {} of {}). Retrying in {} ms. URL: {}. Cause: {}", - attempt, MAX_HTTP_ATTEMPTS, delayMs, url, e.toString()); + attempt, MAX_HTTP_ATTEMPTS, delayMs, redactApiKey(url, apiKeyFrom(url)), e.toString()); sleepMs(delayMs); } } @@ -419,7 +441,8 @@ protected String executeGet(String url) throws IOException String body = (status >= 400 && status < 500 && response.getEntity() != null) ? EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8) : null; - throw new HttpResponseException(status, errorDetail(status, response.getReasonPhrase(), body)); + throw new HttpResponseException(status, + errorDetail(status, response.getReasonPhrase(), body, apiKeyFrom(url))); } return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); }); @@ -429,17 +452,46 @@ protected String executeGet(String url) throws IOException /** * Build the message for a non-2xx HttpResponseException. For client errors (4xx) the response * body is appended, so NCBI's reason (e.g. "API key invalid") reaches the log. Other statuses - * use only the reason phrase. + * use only the reason phrase. The body is third-party text on its way to a log, so any + * occurrence of the API key is removed from it. */ - private static String errorDetail(int status, String reasonPhrase, @Nullable String body) + static String errorDetail(int status, String reasonPhrase, @Nullable String body, @Nullable String apiKey) { if (status >= 400 && status < 500 && !StringUtils.isBlank(body)) { - return reasonPhrase + " - " + StringUtils.abbreviate(body.strip(), 500); + return reasonPhrase + " - " + redactApiKey(StringUtils.abbreviate(body.strip(), 500), apiKey); } return reasonPhrase; } + /** + * Replace the NCBI API key wherever it appears in text that is about to be logged. The key is a + * query parameter on every eutils URL. When the reminder job runs, the log is the pipeline job + * log, which is readable by anyone with read access to the folder the job ran in. + */ + static String redactApiKey(@Nullable String text, @Nullable String apiKey) + { + if (text == null) + { + return null; + } + String redacted = text.replaceAll("(api_key=)[^&\\s]*", "$1" + REDACTED); + if (!StringUtils.isBlank(apiKey)) + { + redacted = redacted.replace(apiKey.trim(), REDACTED); + } + return redacted; + } + + /** + * Returns the value of the api_key query parameter in the given URL, or null if there is none. + */ + static @Nullable String apiKeyFrom(String url) + { + Matcher matcher = API_KEY_PARAM.matcher(url); + return matcher.find() ? matcher.group(1) : null; + } + /** * Read timeouts and 5xx responses are transient NCBI failures worth retrying. 4xx and other * errors are permanent. @@ -527,7 +579,7 @@ private Map fetchMetadata(Collection ids, String dat catch (IOException | JSONException e) { log.error("Error fetching {} metadata for IDs: {}", database, ids, e); - return Collections.emptyMap(); + throw new NcbiSearchException("Error fetching " + database + " metadata for IDs: " + ids, e); } } @@ -1547,16 +1599,43 @@ public void testRetryDelayMs() public void testErrorDetail() { // 4xx: the response body is appended so the cause (e.g. an invalid API key) is logged - String detail = errorDetail(400, "Bad Request", "{\"error\":\"API key invalid\"}"); + String detail = errorDetail(400, "Bad Request", "{\"error\":\"API key invalid\"}", null); assertTrue(detail.contains("Bad Request")); assertTrue(detail.contains("API key invalid")); // 5xx: body omitted (uninformative) - assertEquals("Internal Server Error", errorDetail(500, "Internal Server Error", "oops")); + assertEquals("Internal Server Error", errorDetail(500, "Internal Server Error", "oops", null)); // 4xx with blank or null body: just the reason phrase, no trailing separator - assertEquals("Bad Request", errorDetail(400, "Bad Request", "")); - assertEquals("Bad Request", errorDetail(400, "Bad Request", null)); + assertEquals("Bad Request", errorDetail(400, "Bad Request", "", null)); + assertEquals("Bad Request", errorDetail(400, "Bad Request", null, null)); + + // A body that quotes the request back must not carry the key into the log + String echoed = errorDetail(400, "Bad Request", "invalid key SECRET123 for api_key=SECRET123", "SECRET123"); + assertFalse("errorDetail must not put the API key in the message", echoed.contains("SECRET123")); + } + + @Test + public void testRedactApiKey() + { + // The key is stripped from an eutils URL, and the rest of the URL is left intact + String url = "https://eutils.ncbi.nlm.nih.gov/esearch.fcgi?db=pmc&api_key=SECRET123&term=PXD001"; + String redacted = redactApiKey(url, "SECRET123"); + assertFalse("The redacted URL must not contain the key", redacted.contains("SECRET123")); + assertTrue("The redacted URL must keep its other parameters", redacted.contains("term=PXD001")); + assertTrue(redacted.contains("db=pmc")); + + // The key is stripped even when it appears without the api_key= prefix + assertFalse(redactApiKey("rejected key SECRET123", "SECRET123").contains("SECRET123")); + + // A URL with no key is unchanged, and null text stays null + String noKey = "https://eutils.ncbi.nlm.nih.gov/esearch.fcgi?db=pmc&term=PXD001"; + assertEquals(noKey, redactApiKey(noKey, null)); + assertNull(redactApiKey(null, "SECRET123")); + + // The key is recovered from the URL so callers do not have to read the settings + assertEquals("SECRET123", apiKeyFrom(url)); + assertNull(apiKeyFrom(noKey)); } @Test diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java new file mode 100644 index 00000000..0e4a8806 --- /dev/null +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.panoramapublic.ncbi; + +/** + * Thrown when a request to NCBI fails, so that callers can tell a failed search from a search that + * found nothing. Without it a rejected API key and a dataset with no published paper both arrive as + * an empty result. + */ +public class NcbiSearchException extends RuntimeException +{ + public NcbiSearchException(String message, Throwable cause) + { + super(message, cause); + } + + public NcbiSearchException(String message) + { + super(message); + } +} diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index 4b9d8ed6..e4a8b649 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -41,6 +41,7 @@ import org.labkey.panoramapublic.model.Journal; import org.labkey.panoramapublic.model.JournalSubmission; import org.labkey.panoramapublic.ncbi.NcbiPublicationSearchService; +import org.labkey.panoramapublic.ncbi.NcbiSearchException; import org.labkey.panoramapublic.ncbi.PublicationMatch; import org.labkey.panoramapublic.query.DatasetStatusManager; import org.labkey.panoramapublic.query.ExperimentAnnotationsManager; @@ -251,6 +252,12 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp return null; } } + catch (NcbiSearchException e) + { + // A search that could not run must not reset the deferral, which would record it + // as having found no new publication. + throw e; + } catch (Exception e) { log.error("Error re-searching publication for experiment {}: {}", expAnnotations.getId(), e.getMessage(), e); @@ -272,6 +279,12 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp { return NcbiPublicationSearchService.get().searchForPublication(expAnnotations, log); } + catch (NcbiSearchException e) + { + // The caller records this as a failed search. Returning null here would make it + // indistinguishable from a dataset with no published paper. + throw e; + } catch (Exception e) { log.error("Error searching for publication for experiment {}: {}", expAnnotations.getId(), e.getMessage(), e); @@ -384,8 +397,17 @@ private void processExperiment(Integer experimentAnnotationsId, ProcessingContex return; } - // Check for publications if enabled - PublicationMatch publicationResult = searchForPublication(expAnnotations, context.getSettings(), _forcePublicationCheck, getUser(), context.isTestMode(), processingResults._log); + // Check for publications if enabled. A search that could not run still gets a reminder, since + // the reminder is about the data being private, not about the paper. + PublicationMatch publicationResult = null; + try + { + publicationResult = searchForPublication(expAnnotations, context.getSettings(), _forcePublicationCheck, getUser(), context.isTestMode(), processingResults._log); + } + catch (NcbiSearchException e) + { + processingResults.addPublicationSearchFailed(experimentAnnotationsId, e); + } if (!context.isTestMode()) { @@ -662,6 +684,7 @@ private static class ProcessingResults private final List _submissionNotFound = new ArrayList<>(); private final List _announcementNotFound = new ArrayList<>(); private final List _submitterNotFound = new ArrayList<>(); + private final List _publicationSearchFailed = new ArrayList<>(); private final List _skipped = new ArrayList<>(); private int _processed = 0; private final int _total; @@ -702,6 +725,12 @@ public void addSubmitterNotFound(Integer experimentId) _log.error("Could not find a submitter user for experiment Id: {}.", experimentId); } + public void addPublicationSearchFailed(Integer experimentId, Exception e) + { + _publicationSearchFailed.add(experimentId); + _log.error("Publication search failed for experiment Id: {}. A reminder was still posted. {}", experimentId, e.getMessage(), e); + } + public void addSkipped(Integer experimentId, ReminderDecision decision) { _skipped.add(experimentId); @@ -739,6 +768,11 @@ public void logSkipped(Logger log) log.error("Support message threads were not found for the following experiment Ids: {}", StringUtils.join(_announcementNotFound, ", ")); } + if (!_publicationSearchFailed.isEmpty()) + { + log.error("Publication search failed for the following experiment Ids: {}. Check the NCBI API key in the Private Data Reminder Settings.", StringUtils.join(_publicationSearchFailed, ", ")); + } + if (!_submitterNotFound.isEmpty()) { log.error("Submitter user was not found for the following experiment Ids: {}", StringUtils.join(_submitterNotFound, ", ")); diff --git a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp index 95f311fa..4a161bfa 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp +++ b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp @@ -81,6 +81,58 @@ } window.location = LABKEY.ActionURL.buildURL("panoramapublic", "searchPublications.view", folderPath); } + + function showValidationResult(result, message, detail) + { + while (result.firstChild) + { + result.removeChild(result.firstChild); + } + result.appendChild(document.createTextNode(message)); + + if (!detail) + { + return; + } + + result.appendChild(document.createTextNode(" ")); + const link = document.createElement("a"); + link.href = "#"; + link.textContent = "Details"; + // The handler is attached here rather than with an onclick attribute, which the Content + // Security Policy blocks. NCBI's reply is encoded because it is third party text. + link.addEventListener("click", function (e) + { + e.preventDefault(); + Ext4.Msg.alert("NCBI response", Ext4.String.htmlEncode(detail)); + }); + result.appendChild(link); + } + + function validateNcbiApiKey() + { + const input = document.getElementsByName("ncbiApiKey")[0]; + const result = document.getElementById("ncbiApiKeyValidationResult"); + result.style.color = ""; + result.textContent = "Checking with NCBI..."; + + LABKEY.Ajax.request({ + url: LABKEY.ActionURL.buildURL("panoramapublic", "validateNcbiApiKey.api"), + method: "POST", + // An empty value asks the server to check the saved key, which this form never displays. + jsonData: {ncbiApiKey: input ? input.value : ""}, + success: LABKEY.Utils.getCallbackWrapper(function (response) + { + result.style.color = response.valid ? "green" : "red"; + showValidationResult(result, response.message, response.detail); + }), + failure: LABKEY.Utils.getCallbackWrapper(function () + { + result.style.color = "red"; + showValidationResult(result, "Could not reach the server to validate the key.", null); + }) + }); + } @@ -175,11 +227,17 @@ <%=h(PrivateDataReminderSettings.PROP_NCBI_API_KEY)%> - + " /> + <%=button("Validate").onClick("validateNcbiApiKey(); return false;")%> +
Optional. An NCBI API key raises the request rate limit for PubMed/PMC searches from 3 to 10 per second.
- Create one under Account settings at ncbi.nlm.nih.gov. Leave blank to search without a key. + Create one under Account settings at ncbi.nlm.nih.gov. The saved key is not displayed. Leaving this + blank keeps the key that is already saved. +
+
diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index 3637108c..425860c4 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -645,7 +645,9 @@ protected Map getPrivateDataReminderSettings() settings.put("reminderFrequency", getFormElement(Locator.input("reminderFrequency"))); settings.put("enablePublicationSearch", String.valueOf(Locator.checkboxByName("enablePublicationSearch").findElement(getDriver()).isSelected())); settings.put("publicationSearchFrequency", getFormElement(Locator.input("publicationSearchFrequency"))); - settings.put("ncbiApiKey", getFormElement(Locator.input("ncbiApiKey"))); + // The saved key is never displayed. The placeholder is the only signal that one is stored. + settings.put("ncbiApiKeySaved", String.valueOf( + Locator.input("ncbiApiKey").findElement(getDriver()).getDomAttribute("placeholder").startsWith("A key is saved"))); return settings; } @@ -694,8 +696,8 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de Locator.checkboxByName("enablePublicationSearch").findElement(getDriver()).isSelected()); if (ncbiApiKey != null) { - assertEquals("The saved NCBI API key should be displayed on the form", ncbiApiKey, - getFormElement(Locator.input("ncbiApiKey"))); + assertEquals("The form should report that a key is saved", "true", + getPrivateDataReminderSettings().get("ncbiApiKeySaved")); } } diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java index 898f64b7..2bd4834a 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java @@ -259,32 +259,44 @@ public void testPublicationSearchAndDismiss() assertNotNull("Expected lastReminderDate for dataset 2", dsStatus2AfterPost.get("LastReminderDate")); assertNotNull("Expected citation to be cached for dataset 2", dsStatus2AfterPost.get("Citation")); - // Verify the NCBI API key setting round-trips (set -> save -> re-read). + verifyNcbiApiKeySettings(); + } + + /** + * The saved key is a credential, so the form reports only whether one is stored. Guards the two + * ways that has gone wrong, displaying the key and erasing it when the field is left blank. + */ + private void verifyNcbiApiKeySettings() + { + if (Boolean.parseBoolean(_originalReminderSettings.get("ncbiApiKeySaved"))) + { + // A saved key cannot be read back, so a test that overwrote it could not put it back. + log("An NCBI API key is already saved on this server. Skipping the key settings checks."); + return; + } + savePrivateDataReminderSettings("2", "0", "0", true, "test-ncbi-api-key"); + assertEquals("The key itself must never be rendered into the form", "", + getFormElement(Locator.input("ncbiApiKey"))); + + // Saving with the field left blank must keep the stored key. Every other field on this page + // is edited routinely, so a blank field cannot mean "remove the key". + savePrivateDataReminderSettings("3", "0", "0", true, ""); + assertEquals("A blank key field must leave the saved key alone", "true", + getPrivateDataReminderSettings().get("ncbiApiKeySaved")); - // Verify the configured key reaches the live eutils requests. NCBI rejects an invalid key - // with HTTP 400, so re-searching dataset 2, which found a publication above, should now - // find nothing. The mock service bypasses the key, so this runs only against real NCBI. if (!_useMockNcbi) { - // Capture the server error count immediately before the deliberate bad-key search so the - // assertion below only counts errors due to the bad-key search. - int serverErrorCount = getServerErrorCount(); - - searchPublicationsForDataset(panoramaPublicProject, TARGET_FOLDER_2, exptId2); - assertTextPresent("No publications found for this dataset."); - assertTextNotPresent(PMID_2); - - // The invalid key makes NCBI return HTTP 400, and errorDetail appends the response body - // to the logged message. - assertTrue("Server log should record NCBI's invalid-key error", - getServerErrors().contains("API key invalid")); - - // The bad-key search logs one error per failed NCBI call. Dataset 2 runs two PMC - // strategy searches, on ProteomeXchange ID and Panorama URL, plus the PubMed fallback. - // checkExpectedErrors clears exactly that many and fails on any others. - checkExpectedErrors(serverErrorCount + 3); + // Only real NCBI can reject a key. The mock never sends one. + click(Locator.tagWithClass("button", "labkey-button").withText("Validate")); + waitForElement(Locator.id("ncbiApiKeyValidationResult").containing("NCBI rejected this key")); } + + // Removing the key takes the explicit checkbox. + checkCheckbox(Locator.checkboxByName("clearNcbiApiKey")); + clickButton("Save"); + assertEquals("The saved key should be gone after Remove the saved key", "false", + getPrivateDataReminderSettings().get("ncbiApiKeySaved")); } /* @@ -515,8 +527,10 @@ public void resetAfterTest() _originalReminderSettings.get("extensionLength"), _originalReminderSettings.get("delayUntilFirstReminder"), _originalReminderSettings.get("reminderFrequency"), + // The test removes the key it saved, and a key saved before the run cannot be + // read back to restore it, so leave the key field alone here. Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch")), - _originalReminderSettings.get("ncbiApiKey")); + null); } } From 759f5d94899745671b5136521f803439062445b7 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Mon, 31 Aug 2026 16:35:54 -0700 Subject: [PATCH 07/24] Moved the NCBI API key coverage into its own Selenium test * Added NcbiApiKeyTest, which fails with instructions rather than skipping when a key is already saved * Covered the Validate button on TeamCity through the mock NCBI service, so nothing is skipped there * Removed the key coverage from PublicationSearchTest, which no longer saves or removes a site-wide key * Removed the test's own key in @After so a failed run cannot leave one that breaks every later search Co-Authored-By: Claude --- .../tests/panoramapublic/NcbiApiKeyTest.java | 153 ++++++++++++++++++ .../panoramapublic/PublicationSearchTest.java | 42 +---- 2 files changed, 156 insertions(+), 39 deletions(-) create mode 100644 panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java new file mode 100644 index 00000000..b0c522db --- /dev/null +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.test.tests.panoramapublic; + +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.SimplePostCommand; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.Locator; +import org.labkey.test.TestProperties; +import org.labkey.test.categories.External; +import org.labkey.test.categories.MacCossLabModules; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Covers the NCBI API key on the Private Data Reminder Settings page. The key is a credential, so + * the page stores it in the encrypted property store and never displays it again. + * + * The key is site wide and cannot be read back, so this test cannot save one without destroying + * whatever is already there. It fails rather than skips when a key is saved, so the missing + * coverage is visible instead of silent. + */ +@Category({External.class, MacCossLabModules.class}) +@BaseWebDriverTest.ClassTimeout(minutes = 5) +public class NcbiApiKeyTest extends PanoramaPublicBaseTest +{ + private static final String TEST_API_KEY = "test-ncbi-api-key"; + + private boolean _savedTestApiKey = false; + private boolean _useMockNcbi = false; + + @Test + public void testNcbiApiKeySettings() + { + setupMockNcbiService(); + + assertEquals("An NCBI API key is saved on this server. This test saves its own key and cannot" + + " restore yours, because a saved key is never readable. Remove the key on the" + + " Private Data Reminder Settings page, run this test, then enter the key again.", + "false", getPrivateDataReminderSettings().get("ncbiApiKeySaved")); + + _savedTestApiKey = true; + savePrivateDataReminderSettings("2", "0", "0", true, TEST_API_KEY); + + assertEquals("The saved key must never be rendered into the form", "", + getFormElement(Locator.input("ncbiApiKey"))); + + // Saving with the field left blank must keep the stored key. Every other field on this page + // is edited routinely, so a blank field cannot mean "remove the key". + savePrivateDataReminderSettings("3", "0", "0", true, ""); + assertEquals("A blank key field must leave the saved key alone", "true", + getPrivateDataReminderSettings().get("ncbiApiKeySaved")); + + verifyValidateButton(); + + // Removing a key takes the explicit checkbox. + checkCheckbox(Locator.checkboxByName("clearNcbiApiKey")); + clickButton("Save"); + _savedTestApiKey = false; + assertEquals("Remove the saved key should remove it", "false", + getPrivateDataReminderSettings().get("ncbiApiKeySaved")); + } + + /** + * Covers the button, the request it sends and the message it displays. The mock answers every + * request, so it reports the key as accepted. Only real NCBI rejects one. + */ + private void verifyValidateButton() + { + setFormElement(Locator.input("ncbiApiKey"), "not-a-real-key"); + click(Locator.lkButton("Validate")); + + String expected = _useMockNcbi ? "NCBI accepted this key" : "NCBI rejected this key"; + waitForElement(Locator.id("ncbiApiKeyValidationResult").containing(expected)); + + setFormElement(Locator.input("ncbiApiKey"), ""); + } + + /* + * On TeamCity, route NCBI requests through the mock so this test does not depend on NCBI being + * reachable. On a development machine, use the real service so a key can actually be rejected. + */ + private void setupMockNcbiService() + { + if (!TestProperties.isTestRunningOnTeamCity()) + { + return; + } + + try + { + SimplePostCommand command = new SimplePostCommand("panoramapublic", "setupMockNcbiService"); + command.execute(createDefaultConnection(), "/"); + _useMockNcbi = true; + log("Using mock NCBI service"); + } + catch (IOException | CommandException e) + { + fail("Failed to set up the mock NCBI service: " + e.getMessage()); + } + } + + private void restoreNcbiService() + { + try + { + SimplePostCommand command = new SimplePostCommand("panoramapublic", "restoreNcbiService"); + command.execute(createDefaultConnection(), "/"); + log("Restored real NCBI service"); + } + catch (IOException | CommandException e) + { + log("Warning: Failed to restore NCBI service: " + e.getMessage()); + } + } + + @After + public void removeTestApiKey() + { + if (_useMockNcbi) + { + restoreNcbiService(); + } + + if (_savedTestApiKey) + { + // Remove the key even when the test failed before its own removal step. A key NCBI + // rejects makes every publication search on this server fail, including the next run's. + getPrivateDataReminderSettings(); + checkCheckbox(Locator.checkboxByName("clearNcbiApiKey")); + clickButton("Save"); + } + } +} diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java index 2bd4834a..2a6bff8c 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java @@ -259,45 +259,8 @@ public void testPublicationSearchAndDismiss() assertNotNull("Expected lastReminderDate for dataset 2", dsStatus2AfterPost.get("LastReminderDate")); assertNotNull("Expected citation to be cached for dataset 2", dsStatus2AfterPost.get("Citation")); - verifyNcbiApiKeySettings(); } - /** - * The saved key is a credential, so the form reports only whether one is stored. Guards the two - * ways that has gone wrong, displaying the key and erasing it when the field is left blank. - */ - private void verifyNcbiApiKeySettings() - { - if (Boolean.parseBoolean(_originalReminderSettings.get("ncbiApiKeySaved"))) - { - // A saved key cannot be read back, so a test that overwrote it could not put it back. - log("An NCBI API key is already saved on this server. Skipping the key settings checks."); - return; - } - - savePrivateDataReminderSettings("2", "0", "0", true, "test-ncbi-api-key"); - assertEquals("The key itself must never be rendered into the form", "", - getFormElement(Locator.input("ncbiApiKey"))); - - // Saving with the field left blank must keep the stored key. Every other field on this page - // is edited routinely, so a blank field cannot mean "remove the key". - savePrivateDataReminderSettings("3", "0", "0", true, ""); - assertEquals("A blank key field must leave the saved key alone", "true", - getPrivateDataReminderSettings().get("ncbiApiKeySaved")); - - if (!_useMockNcbi) - { - // Only real NCBI can reject a key. The mock never sends one. - click(Locator.tagWithClass("button", "labkey-button").withText("Validate")); - waitForElement(Locator.id("ncbiApiKeyValidationResult").containing("NCBI rejected this key")); - } - - // Removing the key takes the explicit checkbox. - checkCheckbox(Locator.checkboxByName("clearNcbiApiKey")); - clickButton("Save"); - assertEquals("The saved key should be gone after Remove the saved key", "false", - getPrivateDataReminderSettings().get("ncbiApiKeySaved")); - } /* * Navigate to the Panorama Public copy folder and get the experiment ID. @@ -527,11 +490,12 @@ public void resetAfterTest() _originalReminderSettings.get("extensionLength"), _originalReminderSettings.get("delayUntilFirstReminder"), _originalReminderSettings.get("reminderFrequency"), - // The test removes the key it saved, and a key saved before the run cannot be - // read back to restore it, so leave the key field alone here. + // A key saved before the run cannot be read back to restore it, so leave the + // key field alone here. Any key the test saved is removed below. Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch")), null); } + } @Override From 2a67949a9c4904546ad8f8b4d5f9c3afb3005131 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Mon, 31 Aug 2026 17:06:53 -0700 Subject: [PATCH 08/24] Hardened the NCBI retry loop against 429, interruption and unreadable error bodies * Retried 429, which is NCBI's answer when the request rate is exceeded and was failing fast as a 4xx * Disabled HttpClient's own retries, which doubled the requests MAX_HTTP_ATTEMPTS names on a persistent 503 * Stopped retrying once the thread is interrupted, since every later sleep throws at once and drops the backoff * Stopped the reminder job starting another experiment after interruption, which would run with no rate limit * Bounded the error body read and caught its ParseException, which could otherwise discard the HTTP status Co-Authored-By: Claude --- .../NcbiPublicationSearchServiceImpl.java | 103 +++++++++++++++--- .../pipeline/PrivateDataReminderJob.java | 8 ++ 2 files changed, 98 insertions(+), 13 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index a7dae20a..2953bacc 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -24,6 +24,7 @@ import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager; import org.apache.hc.client5.http.HttpResponseException; +import org.apache.hc.core5.http.ClassicHttpResponse; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.util.Timeout; import org.apache.logging.log4j.Logger; @@ -104,6 +105,9 @@ public static void setInstance(NcbiPublicationSearchService impl) private static final int MAX_HTTP_ATTEMPTS = 3; // initial try + 2 retries private static final int RETRY_BASE_DELAY_MS = 500; // exponential backoff base + private static final int TOO_MANY_REQUESTS = 429; // NCBI's answer when the request rate is exceeded + private static final int MAX_ERROR_BODY_CHARS = 500; + private static final String REDACTED = "REDACTED"; private static final Pattern API_KEY_PARAM = Pattern.compile("api_key=([^&\\s]*)"); @@ -406,7 +410,12 @@ protected String getString(String url, Logger log) throws IOException long delayMs = retryDelayMs(attempt); getLog(log).warn("NCBI request failed (attempt {} of {}). Retrying in {} ms. URL: {}. Cause: {}", attempt, MAX_HTTP_ATTEMPTS, delayMs, redactApiKey(url, apiKeyFrom(url)), e.toString()); - sleepMs(delayMs); + if (!sleepMs(delayMs)) + { + // The thread was interrupted. Retrying now would run the remaining attempts with + // no delay, because every later sleep throws at once. + throw e; + } } } } @@ -429,6 +438,9 @@ protected String executeGet(String url) throws IOException try (CloseableHttpClient client = HttpClientBuilder.create() .setDefaultRequestConfig(requestConfig) .setConnectionManager(connectionManager) + // HttpClient retries 429 and 503 once on its own, which would make MAX_HTTP_ATTEMPTS + // cost twice the requests it names. getString is the only retry. + .disableAutomaticRetries() .build()) { HttpGet getRequest = new HttpGet(url); @@ -436,19 +448,38 @@ protected String executeGet(String url) throws IOException int status = response.getCode(); if (status < 200 || status >= 300) { - // 5xx bodies are large, uninformative HTML error pages, so only client error - // bodies are read. - String body = (status >= 400 && status < 500 && response.getEntity() != null) - ? EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8) - : null; throw new HttpResponseException(status, - errorDetail(status, response.getReasonPhrase(), body, apiKeyFrom(url))); + errorDetail(status, response.getReasonPhrase(), readErrorBody(status, response), + apiKeyFrom(url))); } return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); }); } } + /** + * Read the body of a client error response. 5xx bodies are large, uninformative HTML error + * pages, so only client error bodies are read, and only the first {@link #MAX_ERROR_BODY_CHARS} + * characters. A body that cannot be read or parsed returns null, because losing NCBI's text is + * better than losing the status code the caller decides on. + */ + private static @Nullable String readErrorBody(int status, ClassicHttpResponse response) + { + if (status < 400 || status >= 500 || response.getEntity() == null) + { + return null; + } + + try + { + return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8, MAX_ERROR_BODY_CHARS); + } + catch (IOException | org.apache.hc.core5.http.ParseException e) + { + return null; + } + } + /** * Build the message for a non-2xx HttpResponseException. For client errors (4xx) the response * body is appended, so NCBI's reason (e.g. "API key invalid") reaches the log. Other statuses @@ -493,7 +524,8 @@ static String redactApiKey(@Nullable String text, @Nullable String apiKey) } /** - * Read timeouts and 5xx responses are transient NCBI failures worth retrying. 4xx and other + * Read timeouts, 5xx responses and 429 are transient NCBI failures worth retrying. NCBI answers + * 429 when the request rate is exceeded, which is the failure backoff exists for. Other 4xx * errors are permanent. */ private static boolean isRetryable(IOException e) @@ -504,7 +536,7 @@ private static boolean isRetryable(IOException e) } if (e instanceof HttpResponseException hre) { - return hre.getStatusCode() >= 500; + return hre.getStatusCode() >= 500 || hre.getStatusCode() == TOO_MANY_REQUESTS; } return false; } @@ -517,15 +549,21 @@ private static long retryDelayMs(int attempt) return (long) RETRY_BASE_DELAY_MS << (attempt - 1); } - private static void sleepMs(long ms) + /** + * @return false if the thread was interrupted, in which case the caller should stop rather than + * carry on without the delay it asked for. + */ + private static boolean sleepMs(long ms) { try { Thread.sleep(ms); + return true; } catch (InterruptedException e) { Thread.currentThread().interrupt(); + return false; } } @@ -1580,7 +1618,10 @@ public void testIsRetryable() assertTrue(isRetryable(new HttpResponseException(500, "Internal Server Error"))); assertTrue(isRetryable(new HttpResponseException(503, "Service Unavailable"))); - // 4xx and generic IO errors are permanent -> fail fast + // 429 is NCBI's answer when the request rate is exceeded, which backoff is for + assertTrue(isRetryable(new HttpResponseException(429, "Too Many Requests"))); + + // Other 4xx and generic IO errors are permanent -> fail fast assertFalse(isRetryable(new HttpResponseException(400, "Bad Request"))); assertFalse(isRetryable(new HttpResponseException(404, "Not Found"))); assertFalse(isRetryable(new IOException("connection reset"))); @@ -1589,10 +1630,13 @@ public void testIsRetryable() @Test public void testRetryDelayMs() { - // Exponential backoff of 500ms, 1000ms, 2000ms per attempt. + // Exponential backoff of 500ms then 1000ms. With MAX_HTTP_ATTEMPTS at 3 the loop sleeps + // after the first two failures and rethrows after the third, so those are the only + // delays a request can wait. assertEquals(500, retryDelayMs(1)); assertEquals(1000, retryDelayMs(2)); - assertEquals(2000, retryDelayMs(3)); + assertEquals("A request sleeps once per failed attempt except the last, so only the" + + " delays asserted above are reachable", 2, MAX_HTTP_ATTEMPTS - 1); } @Test @@ -1676,6 +1720,39 @@ protected String executeGet(String url) throws IOException assertEquals("Should retry until the 3rd attempt succeeds", 3, attempts[0]); } + @Test + public void testGetStringStopsWhenInterrupted() + { + // An interrupted thread cannot wait, so retrying would send the remaining attempts back + // to back. getString should give up after the first failure instead. + int[] attempts = {0}; + NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() + { + @Override + protected String executeGet(String url) throws IOException + { + attempts[0]++; + Thread.currentThread().interrupt(); + throw new HttpResponseException(503, "Service Unavailable"); + } + }; + + try + { + service.getString("http://test", LOG); + fail("Expected the interrupted request to be rethrown"); + } + catch (IOException expected) + { + assertEquals("An interrupted request should not be retried", 1, attempts[0]); + } + finally + { + // Clear the flag so it cannot affect the tests that run after this one. + Thread.interrupted(); + } + } + @Test public void testGetStringGivesUpAfterMaxAttempts() { diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index e4a8b649..282ae46a 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -341,6 +341,14 @@ private void processExperiments(List expAnnotationIds, ProcessingContex } for (Integer experimentAnnotationsId : exptIds) { + if (Thread.currentThread().isInterrupted()) + { + // Cancelling the job clears the NCBI rate limiter, because every sleep from here on + // throws at once. Stop instead of running the rest at full speed. + log.warn("Job was interrupted. Stopping before experiment {}.", experimentAnnotationsId); + break; + } + try (DbScope.Transaction transaction = PanoramaPublicManager.getSchema().getScope().ensureTransaction()) { processExperiment(experimentAnnotationsId, context, processingResults); From 89677df542d5ba608eab32d075bc759e598b9095 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Mon, 31 Aug 2026 17:10:07 -0700 Subject: [PATCH 09/24] Fixed two date tests that failed on the last days of a month * Took the current date from the expiry the settings calculate, rather than from today plus an offset * On 31 August, six months back then forward again gives 28 August, so the expiry fell before the date checked * Removed the currentDate and minutesOffset parameters that the change left always null and zero * Left the production arithmetic alone, since adding months and clamping to a shorter month is correct * Corrected two test comments, one naming a map key that was renamed and one naming the wrong cleanup method Co-Authored-By: Claude --- .../message/PrivateDataReminderSettings.java | 108 ++++++++++++------ .../PanoramaPublicBaseTest.java | 2 +- .../panoramapublic/PublicationSearchTest.java | 4 +- 3 files changed, 76 insertions(+), 38 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java index 9453b70d..376e25e5 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java +++ b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java @@ -409,92 +409,130 @@ public void testIsLastReminderRecentScenarios() private void testExtensionIsValid(PrivateDataReminderSettings settings, int monthsOffset) { - testExtensionIsValid(settings, monthsOffset, 0, null, true); + testExtensionIsValid(settings, monthsOffset, true); } private void testExtensionIsExpired(PrivateDataReminderSettings settings, int monthsOffset) { - testExtensionIsValid(settings, monthsOffset, 0, null, false); + testExtensionIsValid(settings, monthsOffset, false); } private void testExtensionIsValidAsOf(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset) { - testExtensionIsValid(settings, monthsOffset, minutesOffset, dateFromNow(), true); + testExtensionAtExpiry(settings, monthsOffset, minutesOffset, true); } - private void testExtensionIsExpiredAsOf(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset) + /** + * Checks the boundary at the moment an extension expires. The current date comes from the + * expiry the settings calculate, not from today. Subtracting months and adding them back does + * not always return to the same day, because a shorter target month clamps the day, so a date + * built from today can sit on the wrong side of the boundary. On 31 August, subtracting six + * months gives 28 February and adding six back gives 28 August. + * + * @param minutesBeforeExpiry how long before the expiry to check. Zero is the expiry itself, + * and a negative value is after it. + */ + private void testExtensionAtExpiry(PrivateDataReminderSettings settings, int monthsOffset, + int minutesBeforeExpiry, boolean expectedValid) { - testExtensionIsValid(settings, monthsOffset, minutesOffset, dateFromNow(), false); + DatasetStatus datasetStatus = new DatasetStatus(); + datasetStatus.setExtensionRequestedDate(dateFromNow(monthsOffset, 0, 0)); + + Date expiry = settings.getExtensionValidUntilDate(datasetStatus); + Date currentDate = Date.from(expiry.toInstant().minusSeconds(minutesBeforeExpiry * 60L)); + + String failureMessage = String.format( + "Extension is %s; Extension Length: %d; Extension Requested On: %s; Valid Until: %s; Current Date: %s", + expectedValid ? "valid" : "expired", settings.getExtensionLength(), + datasetStatus.getExtensionRequestedDate(), expiry, currentDate); + + boolean isValid = settings.isExtensionValidAsOf(datasetStatus, currentDate); + if (expectedValid) assertTrue(failureMessage, isValid); + else assertFalse(failureMessage, isValid); } - private void testExtensionIsValid(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset, - Date currentDate, boolean expectedValid) + private void testExtensionIsExpiredAsOf(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset) { - Date extensionDate = dateFromNow(monthsOffset, 0, minutesOffset); + testExtensionAtExpiry(settings, monthsOffset, minutesOffset, false); + } + private void testExtensionIsValid(PrivateDataReminderSettings settings, int monthsOffset, boolean expectedValid) + { DatasetStatus datasetStatus = new DatasetStatus(); - datasetStatus.setExtensionRequestedDate(extensionDate); + datasetStatus.setExtensionRequestedDate(dateFromNow(monthsOffset, 0, 0)); String failureMessage = String.format("Extension is %s; Extension Length: %d; Extension Requested On: %s; Valid Until: %s", expectedValid ? "valid" : "expired", settings.getExtensionLength(), datasetStatus.getExtensionRequestedDate(), settings.getExtensionValidUntilDate(datasetStatus)); - if (currentDate != null) - { - failureMessage += String.format("; Current Date: %s", currentDate); - } - - boolean isValid = currentDate == null - ? settings.isExtensionValid(datasetStatus) - : settings.isExtensionValidAsOf(datasetStatus, currentDate); + + boolean isValid = settings.isExtensionValid(datasetStatus); if (expectedValid) assertTrue(failureMessage, isValid); else assertFalse(failureMessage, isValid); } private void testReminderIsRecent(PrivateDataReminderSettings settings, int daysOffset) { - testReminderIsRecent(settings, 0, daysOffset, 0, null, true); + testReminderIsRecent(settings, daysOffset, true); } private void testReminderIsOld(PrivateDataReminderSettings settings, int daysOffset) { - testReminderIsRecent(settings, 0, daysOffset, 0, null, false); + testReminderIsRecent(settings, daysOffset, false); } private void testReminderIsRecentAsOf(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset) { - testReminderIsRecent(settings, monthsOffset, 0, minutesOffset, dateFromNow(), true); + testReminderAtExpiry(settings, monthsOffset, minutesOffset, true); } private void testReminderIsOldAsOf(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset) { - testReminderIsRecent(settings, monthsOffset, 0, minutesOffset, dateFromNow(), false); + testReminderAtExpiry(settings, monthsOffset, minutesOffset, false); } - private void testReminderIsRecent(PrivateDataReminderSettings settings, int monthsOffset, int daysOffset, int minutesOffset, - Date currentDate, boolean expectedRecent) + /** + * Checks the boundary at the moment a reminder stops counting as recent. The current date + * comes from the date the settings calculate, for the reason given on + * {@link #testExtensionAtExpiry}. + * + * @param minutesBeforeExpiry how long before that date to check. Zero is the date itself, and + * a negative value is after it. + */ + private void testReminderAtExpiry(PrivateDataReminderSettings settings, int monthsOffset, + int minutesBeforeExpiry, boolean expectedRecent) { - Date reminderDate = dateFromNow(monthsOffset, daysOffset, minutesOffset); + DatasetStatus datasetStatus = new DatasetStatus(); + datasetStatus.setLastReminderDate(dateFromNow(monthsOffset, 0, 0)); + + Date expiry = settings.getReminderValidUntilDate(datasetStatus); + Date currentDate = Date.from(expiry.toInstant().minusSeconds(minutesBeforeExpiry * 60L)); + String failureMessage = String.format( + "Reminder is %s; Reminder Frequency: %d; Reminder Sent On: %s; Valid Until: %s; Current Date: %s", + expectedRecent ? "recent" : "old", settings.getReminderFrequency(), + datasetStatus.getLastReminderDate(), expiry, currentDate); + + boolean isRecent = settings.isLastReminderRecentAsOf(datasetStatus, currentDate); + if (expectedRecent) assertTrue(failureMessage, isRecent); + else assertFalse(failureMessage, isRecent); + } + + private void testReminderIsRecent(PrivateDataReminderSettings settings, int daysOffset, boolean expectedRecent) + { DatasetStatus datasetStatus = new DatasetStatus(); - datasetStatus.setLastReminderDate(reminderDate); + datasetStatus.setLastReminderDate(dateFromNow(0, daysOffset, 0)); String failureMessage = String.format("Reminder is %s; Reminder Frequency: %d; Reminder Sent On: %s; Valid Until: %s", expectedRecent ? "recent" : "old", settings.getReminderFrequency(), datasetStatus.getLastReminderDate(), settings.getReminderValidUntilDate(datasetStatus)); - if (currentDate != null) - { - failureMessage += String.format("; Current Date: %s", currentDate); - } - - boolean isValid = currentDate == null - ? settings.isLastReminderRecent(datasetStatus) - : settings.isLastReminderRecentAsOf(datasetStatus, currentDate); - if (expectedRecent) assertTrue(failureMessage, isValid); - else assertFalse(failureMessage, isValid); + + boolean isRecent = settings.isLastReminderRecent(datasetStatus); + if (expectedRecent) assertTrue(failureMessage, isRecent); + else assertFalse(failureMessage, isRecent); } private PrivateDataReminderSettings createTestSettingsExtensionLength(int extensionLength) diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index 425860c4..679c51f9 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -631,7 +631,7 @@ protected void verifyIsPublicColumn(String panoramaPublicProject, String experim /** * Navigate to the Private Data Reminder Settings page and read the current form values. - * Returns a map with keys: extensionLength, delayUntilFirstReminder, reminderFrequency, enablePublicationSearch, publicationSearchFrequency, ncbiApiKey. + * Returns a map with keys: extensionLength, delayUntilFirstReminder, reminderFrequency, enablePublicationSearch, publicationSearchFrequency, ncbiApiKeySaved. */ protected Map getPrivateDataReminderSettings() { diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java index 2a6bff8c..25124e57 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java @@ -114,8 +114,8 @@ public void testPublicationSearchAndDismiss() // Step 1: Set up mock NCBI service if running on TeamCity setupMockNcbiService(); - // Capture the existing reminder settings up front so doCleanup can restore them exactly. - // The dev machine may already have a real NCBI API key (and other non-default values) set. + // Capture the existing reminder settings up front so resetAfterTest can put them back. A dev + // machine may have non-default values set. _originalReminderSettings = getPrivateDataReminderSettings(); // Step 2: Create dataset 1 folder, submit to Panorama Public, and copy From 2bf6107e6c8bac695b67e95b3d557bf5fb0d23f5 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Mon, 31 Aug 2026 19:36:01 -0700 Subject: [PATCH 10/24] Checked the NCBI API key before the reminder job posts anything * PrivateDataReminderJob checks a configured key before any dataset, and errors without posting when NCBI rejects it * checkApiKey replaces validateApiKey and reports VALID, REJECTED or UNCONFIRMED, so an NCBI outage does not stop the run * The Validate button now reports a request that never reached NCBI separately from a rejected key * The end-of-run message no longer tells an admin to check the key when the failure was something else * Added a unit test for the classification, driven through an executeGet override so it needs no network Co-Authored-By: Claude --- .../PanoramaPublicController.java | 16 ++-- .../panoramapublic/ncbi/NcbiApiKeyCheck.java | 74 +++++++++++++++++++ .../ncbi/NcbiPublicationSearchService.java | 3 +- .../NcbiPublicationSearchServiceImpl.java | 53 ++++++++++++- .../pipeline/PrivateDataReminderJob.java | 44 ++++++++++- 5 files changed, 178 insertions(+), 12 deletions(-) create mode 100644 panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiApiKeyCheck.java diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java index c3d0ed24..d5c6b32f 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java @@ -158,6 +158,7 @@ import org.labkey.panoramapublic.message.PrivateDataMessageScheduler; import org.labkey.panoramapublic.message.PrivateDataReminderSettings; import org.labkey.panoramapublic.ncbi.MockNcbiPublicationSearchService; +import org.labkey.panoramapublic.ncbi.NcbiApiKeyCheck; import org.labkey.panoramapublic.ncbi.NcbiPublicationSearchService; import org.labkey.panoramapublic.ncbi.NcbiPublicationSearchServiceImpl; import org.labkey.panoramapublic.ncbi.PublicationMatch; @@ -10107,9 +10108,9 @@ public Object execute(PrivateDataReminderSettingsForm form, BindException errors return response; } - String error = NcbiPublicationSearchService.get().validateApiKey(apiKey); - response.put("valid", error == null); - if (error == null) + NcbiApiKeyCheck check = NcbiPublicationSearchService.get().checkApiKey(apiKey); + response.put("valid", check.isValid()); + if (check.isValid()) { // Validating does not store anything, so say so. Otherwise "accepted" reads as // confirmation that the key is now in effect. @@ -10122,9 +10123,12 @@ public Object execute(PrivateDataReminderSettingsForm form, BindException errors { // The short message goes beside the field. NCBI's own words are offered separately, // since the admin holding the key is the one who has to act on them. - response.put("message", "NCBI rejected this key."); - response.put("detail", error); - LOG.warn("NCBI rejected an API key entered on the Private Data Reminder Settings page. {}", error); + response.put("message", check.isRejected() + ? "NCBI rejected this key." + : "Could not reach NCBI to check this key."); + response.put("detail", check.getMessage()); + LOG.warn("Could not confirm an API key entered on the Private Data Reminder Settings page. {}", + check.getMessage()); } return response; } diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiApiKeyCheck.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiApiKeyCheck.java new file mode 100644 index 00000000..b7bcf95e --- /dev/null +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiApiKeyCheck.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.panoramapublic.ncbi; + +import org.jetbrains.annotations.Nullable; + +/** + * The outcome of checking an NCBI API key. A rejected key is a configuration problem an admin has to + * correct. A check that could not be completed is not, so callers need to tell the two apart. + */ +public class NcbiApiKeyCheck +{ + public enum Status { VALID, REJECTED, UNCONFIRMED } + + private final Status _status; + private final String _message; + + private NcbiApiKeyCheck(Status status, @Nullable String message) + { + _status = status; + _message = message; + } + + public static NcbiApiKeyCheck valid() + { + return new NcbiApiKeyCheck(Status.VALID, null); + } + + public static NcbiApiKeyCheck rejected(@Nullable String message) + { + return new NcbiApiKeyCheck(Status.REJECTED, message); + } + + public static NcbiApiKeyCheck unconfirmed(@Nullable String message) + { + return new NcbiApiKeyCheck(Status.UNCONFIRMED, message); + } + + public Status getStatus() + { + return _status; + } + + /** + * @return NCBI's reason, with the API key removed. Null when the key was accepted. + */ + public @Nullable String getMessage() + { + return _message; + } + + public boolean isValid() + { + return _status == Status.VALID; + } + + public boolean isRejected() + { + return _status == Status.REJECTED; + } +} diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java index e0b05baf..b18bd913 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java @@ -40,9 +40,8 @@ static NcbiPublicationSearchService get() /** * Send a minimal request to NCBI with the given key. - * @return null if NCBI accepted the key, otherwise the reason it gave. */ - @Nullable String validateApiKey(@Nullable String apiKey); + @NotNull NcbiApiKeyCheck checkApiKey(@Nullable String apiKey); @Nullable Pair getPubMedLinkAndCitation(String pubmedId); diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index 2953bacc..d0e6fd38 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -361,7 +361,7 @@ private List executeSearch(String query, String database, Logger log) } @Override - public @Nullable String validateApiKey(@Nullable String apiKey) + public @NotNull NcbiApiKeyCheck checkApiKey(@Nullable String apiKey) { // A minimal ESearch request. NCBI answers a rejected key with 400 and a reason, which the // retry loop does not retry, so this returns quickly either way. @@ -369,11 +369,20 @@ private List executeSearch(String query, String database, Logger log) try { getString(url, LOG); - return null; + return NcbiApiKeyCheck.valid(); + } + catch (HttpResponseException e) + { + // NCBI rejects a key it does not recognise with a 4xx. A 5xx says nothing about the key, + // so the caller is told the key could not be checked rather than that it is bad. + String message = redactApiKey(e.getMessage(), apiKey); + return e.getStatusCode() >= 400 && e.getStatusCode() < 500 + ? NcbiApiKeyCheck.rejected(message) + : NcbiApiKeyCheck.unconfirmed(message); } catch (IOException e) { - return redactApiKey(e.getMessage(), apiKey); + return NcbiApiKeyCheck.unconfirmed(redactApiKey(e.getMessage(), apiKey)); } } @@ -1720,6 +1729,44 @@ protected String executeGet(String url) throws IOException assertEquals("Should retry until the 3rd attempt succeeds", 3, attempts[0]); } + @Test + public void testCheckApiKey() + { + // NCBI rejects a key it does not recognise with a 4xx. The reminder job stops for that, + // so a 5xx has to be reported as a check that could not be completed. + assertEquals(NcbiApiKeyCheck.Status.REJECTED, checkApiKeyAgainst(new HttpResponseException(400, "Bad Request")).getStatus()); + assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new HttpResponseException(503, "Service Unavailable")).getStatus()); + assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new SocketTimeoutException("Read timed out")).getStatus()); + assertEquals(NcbiApiKeyCheck.Status.VALID, checkApiKeyAgainst(null).getStatus()); + + // The key must not travel back to the caller in NCBI's reason + NcbiApiKeyCheck rejected = checkApiKeyAgainst( + new HttpResponseException(400, "Bad Request - invalid key SECRET123"), "SECRET123"); + assertFalse("A rejection must not carry the key", rejected.getMessage().contains("SECRET123")); + } + + private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure) + { + return checkApiKeyAgainst(failure, "test-key"); + } + + private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure, String apiKey) + { + NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() + { + @Override + protected String executeGet(String url) throws IOException + { + if (failure != null) + { + throw failure; + } + return "{\"esearchresult\":{\"idlist\":[]}}"; + } + }; + return service.checkApiKey(apiKey); + } + @Test public void testGetStringStopsWhenInterrupted() { diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index 282ae46a..eaf3971f 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -40,6 +40,7 @@ import org.labkey.panoramapublic.model.ExperimentAnnotations; import org.labkey.panoramapublic.model.Journal; import org.labkey.panoramapublic.model.JournalSubmission; +import org.labkey.panoramapublic.ncbi.NcbiApiKeyCheck; import org.labkey.panoramapublic.ncbi.NcbiPublicationSearchService; import org.labkey.panoramapublic.ncbi.NcbiSearchException; import org.labkey.panoramapublic.ncbi.PublicationMatch; @@ -303,11 +304,52 @@ public void run() return; } + if (!ncbiApiKeyAccepted()) + { + setStatus(TaskStatus.error); + return; + } + postMessage(_experimentAnnotationsIds, _panoramaPublic); setStatus(TaskStatus.complete); } + /** + * Check a configured NCBI API key before any dataset is touched. A key NCBI rejects would fail the + * publication search for every dataset, so the job stops with nothing posted. The job goes ahead + * when the check could not be completed, since that says nothing about the key. + */ + private boolean ncbiApiKeyAccepted() + { + PrivateDataReminderSettings settings = PrivateDataReminderSettings.get(); + if (!settings.isEnablePublicationSearch() && !_forcePublicationCheck) + { + return true; + } + + String apiKey = settings.getNcbiApiKey(); + if (StringUtils.isBlank(apiKey)) + { + // Searches run without a key, at NCBI's lower request rate. + return true; + } + + NcbiApiKeyCheck check = NcbiPublicationSearchService.get().checkApiKey(apiKey); + if (check.isRejected()) + { + getLogger().error("NCBI rejected the API key, so no reminders were posted. Correct the key on the Private Data Reminder Settings page and run the job again. {}", + check.getMessage()); + return false; + } + + if (!check.isValid()) + { + getLogger().warn("Could not reach NCBI to check the API key. Continuing. {}", check.getMessage()); + } + return true; + } + private void postMessage(List expAnnotationIds, Journal panoramaPublic) { int total = expAnnotationIds.size(); @@ -778,7 +820,7 @@ public void logSkipped(Logger log) if (!_publicationSearchFailed.isEmpty()) { - log.error("Publication search failed for the following experiment Ids: {}. Check the NCBI API key in the Private Data Reminder Settings.", StringUtils.join(_publicationSearchFailed, ", ")); + log.error("Publication search failed for the following experiment Ids: {}. NCBI's reason is in the error logged for each one.", StringUtils.join(_publicationSearchFailed, ", ")); } if (!_submitterNotFound.isEmpty()) From 4981c02f2f5bf36ceeb0349e4a910baf16c920a3 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Tue, 1 Sep 2026 09:54:43 -0700 Subject: [PATCH 11/24] Made the retry loop's waiting overridable so tests can check the delays * sleepMs and rateLimit became instance methods, so a test subclass can replace the waiting * The retry tests now assert the delays the loop used, 500ms then 1000ms, which retryDelayMs alone cannot show * The 4xx test asserts the loop did not wait at all Co-Authored-By: Claude --- .../NcbiPublicationSearchServiceImpl.java | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index d0e6fd38..3c32b53c 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -562,7 +562,7 @@ private static long retryDelayMs(int attempt) * @return false if the thread was interrupted, in which case the caller should stop rather than * carry on without the delay it asked for. */ - private static boolean sleepMs(long ms) + protected boolean sleepMs(long ms) { try { @@ -1106,7 +1106,7 @@ static List extractTitleKeywords(String title) /** * Rate limiting: wait 400ms between API requests */ - private static void rateLimit() + private void rateLimit() { sleepMs(RATE_LIMIT_DELAY_MS); } @@ -1715,7 +1715,7 @@ public void testGetStringRetriesTransientFailures() throws IOException { // executeGet returns a 5xx twice, then succeeds. getString should retry and return the body. int[] attempts = {0}; - NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() + NoWaitService service = new NoWaitService() { @Override protected String executeGet(String url) throws IOException @@ -1727,6 +1727,23 @@ protected String executeGet(String url) throws IOException }; assertEquals("body", service.getString("http://test", LOG)); assertEquals("Should retry until the 3rd attempt succeeds", 3, attempts[0]); + assertEquals("The loop should wait 500ms then 1000ms", List.of(500L, 1000L), service.sleeps); + } + + /** + * Runs the retry loop without waiting, and records the delays it asked for. A test can then + * check the delays the loop used, which retryDelayMs on its own cannot show. + */ + private static class NoWaitService extends NcbiPublicationSearchServiceImpl + { + private final List sleeps = new ArrayList<>(); + + @Override + protected boolean sleepMs(long ms) + { + sleeps.add(ms); + return true; + } } @Test @@ -1752,7 +1769,7 @@ private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure) private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure, String apiKey) { - NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() + NcbiPublicationSearchServiceImpl service = new NoWaitService() { @Override protected String executeGet(String url) throws IOException @@ -1795,7 +1812,7 @@ protected String executeGet(String url) throws IOException } finally { - // Clear the flag so it cannot affect the tests that run after this one. + // Clear the flag so it cannot reach whatever test runs next. Thread.interrupted(); } } @@ -1805,7 +1822,7 @@ public void testGetStringGivesUpAfterMaxAttempts() { // executeGet always returns a 5xx. getString should try MAX_HTTP_ATTEMPTS times, then rethrow. int[] attempts = {0}; - NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() + NoWaitService service = new NoWaitService() { @Override protected String executeGet(String url) throws IOException @@ -1828,6 +1845,7 @@ protected String executeGet(String url) throws IOException fail("Expected HttpResponseException, got " + e); } assertEquals("Should attempt exactly MAX_HTTP_ATTEMPTS times", MAX_HTTP_ATTEMPTS, attempts[0]); + assertEquals("One wait fewer than attempts, and no wait after the last", List.of(500L, 1000L), service.sleeps); } @Test @@ -1835,7 +1853,7 @@ public void testGetStringDoesNotRetryClientErrors() { // A 4xx is permanent. getString should fail immediately without retrying. int[] attempts = {0}; - NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() + NoWaitService service = new NoWaitService() { @Override protected String executeGet(String url) throws IOException @@ -1858,6 +1876,7 @@ protected String executeGet(String url) throws IOException fail("Expected HttpResponseException, got " + e); } assertEquals("4xx must not be retried", 1, attempts[0]); + assertTrue("A 4xx must not wait", service.sleeps.isEmpty()); } // -- Helper methods for building test JSON -- From 151992f39bec6c0a837ed3163483b72af2652cb8 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Tue, 1 Sep 2026 10:40:47 -0700 Subject: [PATCH 12/24] Stopped a 429 from the API key check looking like a rejected key * checkApiKey reports a 429 as unconfirmed rather than rejected Co-Authored-By: Claude --- .../ncbi/NcbiPublicationSearchServiceImpl.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index 3c32b53c..fc0a14d3 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -363,8 +363,8 @@ private List executeSearch(String query, String database, Logger log) @Override public @NotNull NcbiApiKeyCheck checkApiKey(@Nullable String apiKey) { - // A minimal ESearch request. NCBI answers a rejected key with 400 and a reason, which the - // retry loop does not retry, so this returns quickly either way. + // A minimal ESearch request. A rejected key comes back as a 400 on the first attempt, while + // a 5xx or 429 is retried like any other request before it is called unconfirmed. String url = ESEARCH_URL + "?" + buildCommonParams("pubmed", apiKey) + "&term=labkey&retmax=1&retmode=json"; try { @@ -373,12 +373,12 @@ private List executeSearch(String query, String database, Logger log) } catch (HttpResponseException e) { - // NCBI rejects a key it does not recognise with a 4xx. A 5xx says nothing about the key, - // so the caller is told the key could not be checked rather than that it is bad. + // NCBI rejects a key it does not recognise with a 4xx. A 429 is the request rate and a + // 5xx is NCBI's own failure, so neither counts as a rejection. String message = redactApiKey(e.getMessage(), apiKey); - return e.getStatusCode() >= 400 && e.getStatusCode() < 500 - ? NcbiApiKeyCheck.rejected(message) - : NcbiApiKeyCheck.unconfirmed(message); + boolean rejected = e.getStatusCode() >= 400 && e.getStatusCode() < 500 + && e.getStatusCode() != TOO_MANY_REQUESTS; + return rejected ? NcbiApiKeyCheck.rejected(message) : NcbiApiKeyCheck.unconfirmed(message); } catch (IOException e) { @@ -1753,6 +1753,10 @@ public void testCheckApiKey() // so a 5xx has to be reported as a check that could not be completed. assertEquals(NcbiApiKeyCheck.Status.REJECTED, checkApiKeyAgainst(new HttpResponseException(400, "Bad Request")).getStatus()); assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new HttpResponseException(503, "Service Unavailable")).getStatus()); + + // A 429 is the request rate, not a bad key. Calling it a rejection would stop the + // reminder job and send an admin to correct a key that works. + assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new HttpResponseException(429, "Too Many Requests")).getStatus()); assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new SocketTimeoutException("Read timed out")).getStatus()); assertEquals(NcbiApiKeyCheck.Status.VALID, checkApiKeyAgainst(null).getStatus()); From bf13cb4e7bf793f92adf03005869989361ceb806 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 6 Sep 2026 17:31:22 -0700 Subject: [PATCH 13/24] Addressed review findings on the NCBI publication search and reminder job * A failed NCBI request no longer abandons the remaining PMC strategies or the PubMed fallback * NcbiPublicationSearchServiceImpl.checkApiKey treats only HTTP 400 as a rejected key * PrivateDataReminderJob.run reports cancelled or error instead of complete when it stops early * Failed NCBI requests log at WARN, leaving ERROR for the per-dataset and run summary * NcbiApiKeyTest reads a new data-key-saved attribute rather than the field placeholder * NcbiApiKeyTest restores the settings it overwrites and no longer requires NCBI to reject a key * Corrected comments and javadoc, and small consistency cleanups Co-Authored-By: Claude --- .../panoramapublic/PanoramaPublicModule.java | 1 - .../message/PrivateDataReminderSettings.java | 3 - .../MockNcbiPublicationSearchService.java | 6 +- .../ncbi/NcbiPublicationSearchService.java | 11 +- .../NcbiPublicationSearchServiceImpl.java | 129 +++++++++++++----- .../ncbi/NcbiSearchException.java | 5 +- .../pipeline/PrivateDataReminderJob.java | 44 +++--- .../view/privateDataRemindersSettingsForm.jsp | 6 +- .../tests/panoramapublic/NcbiApiKeyTest.java | 46 ++++++- .../PanoramaPublicBaseTest.java | 6 +- .../panoramapublic/PublicationSearchTest.java | 3 - 11 files changed, 180 insertions(+), 80 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java index 6942ac20..aa8e78df 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java @@ -156,7 +156,6 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) { fileContentService.addFileListener(new PanoramaPublicFileListener()); } - } @Override diff --git a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java index 376e25e5..e1439331 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java +++ b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java @@ -154,9 +154,6 @@ public static void save(PrivateDataReminderSettings settings) settingsMap.put(PROP_REMINDER_TIME, settings.getReminderTimeFormatted()); settingsMap.put(PROP_ENABLE_PUBLICATION_SEARCH, String.valueOf(settings.isEnablePublicationSearch())); settingsMap.put(PROP_PUBLICATION_SEARCH_FREQUENCY, String.valueOf(settings.getPublicationSearchFrequency())); - // The API key is a credential and is saved separately, in the encrypted store. Saving the - // rest of the settings must never change it. - settingsMap.remove(PROP_NCBI_API_KEY); settingsMap.save(); } diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java index 067dabed..f5ad1c14 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java @@ -159,9 +159,9 @@ private JSONObject handleESearch(String url) boolean isPmc = "pmc".equals(extractQueryParam(url, "db")); Map> searchMap = isPmc ? _pmcSearchResults : _pubmedSearchResults; - // Match registered search keys against the decoded ESearch query term, so a key cannot - // match part of another parameter such as tool or email. The real ESearch term wraps the - // key in quotes (e.g. "PXD056793"), so contains() on the term is the right granularity. + // Match search keys against the decoded ESearch query term, so a key cannot match part of + // another parameter such as tool or email. A key is only part of the term - PMC quotes it, + // as in "PXD056793" - so contains() on the term is the right granularity. String term = extractQueryParam(url, "term"); JSONArray idList = new JSONArray(); diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java index b18bd913..d1056ffb 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java @@ -48,16 +48,15 @@ static NcbiPublicationSearchService get() /** * Searches PMC and PubMed for a publication associated with the experiment. * Returns the top match (highest priority) if multiple matches are found, or null if none. - */ - /** - * @throws NcbiSearchException if a request to NCBI fails. An empty result therefore means no - * publication was found, not that the search could not be run. + * + * @throws NcbiSearchException if no publication was found and one or more NCBI requests failed. A + * null result therefore means no publication was found, not that the search could not be run. */ @Nullable PublicationMatch searchForPublication(@NotNull ExperimentAnnotations expAnnotations, @Nullable Logger logger); /** - * @throws NcbiSearchException if a request to NCBI fails. An empty result therefore means no - * publication was found, not that the search could not be run. + * @throws NcbiSearchException if no publication was found and one or more NCBI requests failed. An + * empty result therefore means no publication was found, not that the search could not be run. */ List searchForPublication(@NotNull ExperimentAnnotations expAnnotations, int maxResults, @Nullable Logger logger, boolean getCitations); } diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index fc0a14d3..62cc209c 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -100,12 +100,13 @@ public static void setInstance(NcbiPublicationSearchService impl) private static final int RATE_LIMIT_DELAY_MS = 400; // NCBI allows 3 requests/sec without an API key private static final int TIMEOUT_MS = 10000; // 10 seconds - // NCBI eutils intermittently returns transient 5xx errors and read timeouts, even well under - // the rate limit. Retry those a few times with exponential backoff before giving up. + // NCBI eutils fails intermittently, even well under the rate limit. Retry the transient + // failures listed on isRetryable a few times with exponential backoff before giving up. private static final int MAX_HTTP_ATTEMPTS = 3; // initial try + 2 retries private static final int RETRY_BASE_DELAY_MS = 500; // exponential backoff base - private static final int TOO_MANY_REQUESTS = 429; // NCBI's answer when the request rate is exceeded + private static final int BAD_REQUEST = 400; // NCBI's response when the API key is not recognised + private static final int TOO_MANY_REQUESTS = 429; // NCBI's response when the request rate is exceeded private static final int MAX_ERROR_BODY_CHARS = 500; private static final String REDACTED = "REDACTED"; @@ -177,7 +178,9 @@ private static Logger getLog(@Nullable Logger logger) } catch (IOException e) { - log.error("Error submitting a request to NCBI Literature Citation Exporter. URL: {}", queryUrl, e); + // A missing citation does not fail the match. getCitation returns null and the caller + // displays the publication ID instead. + log.warn("Request to the NCBI Literature Citation Exporter did not complete. URL: {}", queryUrl, e); } return null; } @@ -224,19 +227,34 @@ public List searchForPublication(@NotNull ExperimentAnnotation maxResults = Math.max(1, Math.min(maxResults, NcbiPublicationSearchService.MAX_RESULTS)); log.info("Starting publication search for experiment: {}", expAnnotations.getId()); + // NCBI requests that failed after retries. An empty result is returned only when every request + // completed, so a caller can tell a search that found no publication from one that could not run. + List failedRequests = new ArrayList<>(); + // Search PubMed Central first - List matchedArticles = searchPmc(expAnnotations, log); + List matchedArticles = searchPmc(expAnnotations, log, failedRequests); // If no PMC results, fall back to PubMed if (matchedArticles.isEmpty()) { log.info("No PMC articles found, trying PubMed fallback"); - matchedArticles = searchPubMed(expAnnotations, log); + matchedArticles = searchPubMed(expAnnotations, log, failedRequests); + } + + if (!failedRequests.isEmpty()) + { + log.warn("Some NCBI requests did not complete for experiment {}. Failed requests: {}", + expAnnotations.getId(), StringUtils.join(failedRequests, ", ")); } // Build and return result if (matchedArticles.isEmpty()) { + if (!failedRequests.isEmpty()) + { + throw new NcbiSearchException("Publication search for experiment " + expAnnotations.getId() + + " did not complete. Failed requests: " + StringUtils.join(failedRequests, ", ")); + } log.info("No publications found"); return Collections.emptyList(); } @@ -263,7 +281,8 @@ public List searchForPublication(@NotNull ExperimentAnnotation /** * Search PubMed Central */ - private @NotNull List searchPmc(@NotNull ExperimentAnnotations expAnnotations, Logger log) + private @NotNull List searchPmc(@NotNull ExperimentAnnotations expAnnotations, Logger log, + List failedRequests) { Map searchTermsByStrategy = buildSearchTerms(expAnnotations); @@ -274,11 +293,20 @@ public List searchForPublication(@NotNull ExperimentAnnotation String strategy = entry.getKey(); String searchTerm = entry.getValue(); log.debug("Searching PMC by {}: {}", strategy, searchTerm); - List ids = searchPmc(quote(searchTerm), log); - if (!ids.isEmpty()) + try { - pmcIdsByStrategy.put(strategy, ids); - log.debug("Found {} PMC articles by {}", ids.size(), strategy); + List ids = searchPmc(quote(searchTerm), log); + if (!ids.isEmpty()) + { + pmcIdsByStrategy.put(strategy, ids); + log.debug("Found {} PMC articles by {}", ids.size(), strategy); + } + } + catch (NcbiSearchException e) + { + // NCBI responds inconsistently to the same query, so the remaining strategies are + // still worth running. executeSearch has already logged the query and the cause. + failedRequests.add("PMC search by " + strategy); } rateLimit(); } @@ -291,7 +319,17 @@ public List searchForPublication(@NotNull ExperimentAnnotation log.info("Total unique PMC IDs found: {}", idToStrategies.size()); // Fetch and verify PMC articles - List pmcArticles = fetchAndVerifyPmcArticles(idToStrategies.keySet(), idToStrategies, expAnnotations, log); + List pmcArticles; + try + { + pmcArticles = fetchAndVerifyPmcArticles(idToStrategies.keySet(), idToStrategies, expAnnotations, log); + } + catch (NcbiSearchException e) + { + // The IDs cannot be verified without their metadata, so PMC has no usable result. + failedRequests.add("PMC metadata fetch"); + return Collections.emptyList(); + } // Apply priority filtering return applyPriorityFiltering(pmcArticles, expAnnotations.getCreated(), log); @@ -355,7 +393,9 @@ private List executeSearch(String query, String database, Logger log) } catch (IOException | JSONException e) { - log.error("Error searching {} with query: {}", database, query, e); + // One request, not the outcome for the dataset. The search methods catch this per request + // and carry on with the remaining ones. + log.warn("Search of {} did not complete for query: {}", database, query, e); throw new NcbiSearchException("Error searching " + database + " with query: " + query, e); } } @@ -373,12 +413,12 @@ private List executeSearch(String query, String database, Logger log) } catch (HttpResponseException e) { - // NCBI rejects a key it does not recognise with a 4xx. A 429 is the request rate and a - // 5xx is NCBI's own failure, so neither counts as a rejection. + // NCBI rejects a key it does not recognise with a 400. Any other status says nothing about + // the key, including a 403 or 404 from a proxy between the server and NCBI. String message = redactApiKey(e.getMessage(), apiKey); - boolean rejected = e.getStatusCode() >= 400 && e.getStatusCode() < 500 - && e.getStatusCode() != TOO_MANY_REQUESTS; - return rejected ? NcbiApiKeyCheck.rejected(message) : NcbiApiKeyCheck.unconfirmed(message); + return e.getStatusCode() == BAD_REQUEST + ? NcbiApiKeyCheck.rejected(message) + : NcbiApiKeyCheck.unconfirmed(message); } catch (IOException e) { @@ -421,8 +461,8 @@ protected String getString(String url, Logger log) throws IOException attempt, MAX_HTTP_ATTEMPTS, delayMs, redactApiKey(url, apiKeyFrom(url)), e.toString()); if (!sleepMs(delayMs)) { - // The thread was interrupted. Retrying now would run the remaining attempts with - // no delay, because every later sleep throws at once. + // An interrupted thread cannot wait, so the remaining attempts would run back to + // back. Give up instead. throw e; } } @@ -499,7 +539,7 @@ static String errorDetail(int status, String reasonPhrase, @Nullable String body { if (status >= 400 && status < 500 && !StringUtils.isBlank(body)) { - return reasonPhrase + " - " + redactApiKey(StringUtils.abbreviate(body.strip(), 500), apiKey); + return reasonPhrase + " - " + redactApiKey(StringUtils.abbreviate(body.strip(), MAX_ERROR_BODY_CHARS), apiKey); } return reasonPhrase; } @@ -509,7 +549,7 @@ static String errorDetail(int status, String reasonPhrase, @Nullable String body * query parameter on every eutils URL. When the reminder job runs, the log is the pipeline job * log, which is readable by anyone with read access to the folder the job ran in. */ - static String redactApiKey(@Nullable String text, @Nullable String apiKey) + static @Nullable String redactApiKey(@Nullable String text, @Nullable String apiKey) { if (text == null) { @@ -533,7 +573,7 @@ static String redactApiKey(@Nullable String text, @Nullable String apiKey) } /** - * Read timeouts, 5xx responses and 429 are transient NCBI failures worth retrying. NCBI answers + * Read timeouts, 5xx responses and 429 are transient NCBI failures worth retrying. NCBI returns * 429 when the request rate is exceeded, which is the failure backoff exists for. Other 4xx * errors are permanent. */ @@ -550,9 +590,8 @@ private static boolean isRetryable(IOException e) return false; } - // 500ms after the first failure, then doubling. Jitter is not needed. Both callers, the daily - // reminder job and the UI search, issue NCBI requests sequentially, so a retry never collides - // with a sibling request. + // 500ms after the first failure, then doubling. No jitter. Each caller issues its NCBI requests + // sequentially. Retries collide only when the reminder job and a UI search overlap, which is rare. private static long retryDelayMs(int attempt) { return (long) RETRY_BASE_DELAY_MS << (attempt - 1); @@ -625,7 +664,7 @@ private Map fetchMetadata(Collection ids, String dat } catch (IOException | JSONException e) { - log.error("Error fetching {} metadata for IDs: {}", database, ids, e); + log.warn("Metadata fetch from {} did not complete for IDs: {}", database, ids, e); throw new NcbiSearchException("Error fetching " + database + " metadata for IDs: " + ids, e); } } @@ -766,7 +805,8 @@ private static int countDataIdMatches(PublicationMatch a) /** * Fall back to PubMed search if PMC finds nothing */ - private List searchPubMed(ExperimentAnnotations expAnnotations, Logger log) + private List searchPubMed(ExperimentAnnotations expAnnotations, Logger log, + List failedRequests) { String firstName = expAnnotations.getSubmitterUser() != null ? expAnnotations.getSubmitterUser().getFirstName() : null; @@ -788,7 +828,16 @@ private List searchPubMed(ExperimentAnnotations expAnnotations stripQuerySpecialChars(lastName), stripQuerySpecialChars(firstName), stripQuerySpecialChars(title)); log.debug("PubMed fallback query: {}", query); - List pmids = searchPubMed(query, log); + List pmids; + try + { + pmids = searchPubMed(query, log); + } + catch (NcbiSearchException e) + { + failedRequests.add("PubMed search"); + return Collections.emptyList(); + } if (pmids.isEmpty()) { @@ -799,7 +848,18 @@ private List searchPubMed(ExperimentAnnotations expAnnotations log.info("PubMed fallback found {} result(s), verifying...", pmids.size()); // Fetch metadata and verify - Map metadata = fetchPubMedMetadata(pmids, log); + Map metadata; + try + { + metadata = fetchPubMedMetadata(pmids, log); + } + catch (NcbiSearchException e) + { + // Author and title cannot be verified without the metadata, and an unverified PMID is not + // a match. + failedRequests.add("PubMed metadata fetch"); + return Collections.emptyList(); + } List articles = new ArrayList<>(); for (String pmid : pmids) @@ -1627,7 +1687,7 @@ public void testIsRetryable() assertTrue(isRetryable(new HttpResponseException(500, "Internal Server Error"))); assertTrue(isRetryable(new HttpResponseException(503, "Service Unavailable"))); - // 429 is NCBI's answer when the request rate is exceeded, which backoff is for + // 429 is NCBI's response when the request rate is exceeded, which backoff is for assertTrue(isRetryable(new HttpResponseException(429, "Too Many Requests"))); // Other 4xx and generic IO errors are permanent -> fail fast @@ -1749,11 +1809,16 @@ protected boolean sleepMs(long ms) @Test public void testCheckApiKey() { - // NCBI rejects a key it does not recognise with a 4xx. The reminder job stops for that, + // NCBI rejects a key it does not recognise with a 400. The reminder job stops for that, // so a 5xx has to be reported as a check that could not be completed. assertEquals(NcbiApiKeyCheck.Status.REJECTED, checkApiKeyAgainst(new HttpResponseException(400, "Bad Request")).getStatus()); assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new HttpResponseException(503, "Service Unavailable")).getStatus()); + // A 403 or 404 can come from a proxy between the server and NCBI, which says nothing about + // the key. Calling either a rejection would stop the reminder job on a key that works. + assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new HttpResponseException(403, "Forbidden")).getStatus()); + assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new HttpResponseException(404, "Not Found")).getStatus()); + // A 429 is the request rate, not a bad key. Calling it a rejection would stop the // reminder job and send an admin to correct a key that works. assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new HttpResponseException(429, "Too Many Requests")).getStatus()); diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java index 0e4a8806..907cd0b1 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java @@ -16,9 +16,8 @@ package org.labkey.panoramapublic.ncbi; /** - * Thrown when a request to NCBI fails, so that callers can tell a failed search from a search that - * found nothing. Without it a rejected API key and a dataset with no published paper both arrive as - * an empty result. + * Lets a caller tell a failed NCBI request from a search that found nothing. Without it a rejected + * API key and a dataset with no published paper both arrive as an empty result. */ public class NcbiSearchException extends RuntimeException { diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index eaf3971f..6d4d7b80 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -255,8 +255,7 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp } catch (NcbiSearchException e) { - // A search that could not run must not reset the deferral, which would record it - // as having found no new publication. + // Rethrow so that the caller can record this as a failed NCBI search. throw e; } catch (Exception e) @@ -282,8 +281,7 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp } catch (NcbiSearchException e) { - // The caller records this as a failed search. Returning null here would make it - // indistinguishable from a dataset with no published paper. + // Rethrow so that the caller can record this as a failed NCBI search. throw e; } catch (Exception e) @@ -301,6 +299,7 @@ public void run() if (_panoramaPublic == null) { getLogger().error("Panorama Public project does not exist."); + setStatus(TaskStatus.error); return; } @@ -310,9 +309,7 @@ public void run() return; } - postMessage(_experimentAnnotationsIds, _panoramaPublic); - - setStatus(TaskStatus.complete); + setStatus(postMessage(_experimentAnnotationsIds, _panoramaPublic)); } /** @@ -350,13 +347,17 @@ private boolean ncbiApiKeyAccepted() return true; } - private void postMessage(List expAnnotationIds, Journal panoramaPublic) + /** + * @return complete when every dataset was processed, error when the job could not start, and + * cancelled when it was interrupted partway. + */ + private TaskStatus postMessage(List expAnnotationIds, Journal panoramaPublic) { int total = expAnnotationIds.size(); if (total == 0) { getLogger().info("No private datasets were found."); - return; + return TaskStatus.complete; } Logger log = getLogger(); @@ -364,15 +365,19 @@ private void postMessage(List expAnnotationIds, Journal panoramaPublic) if(!context.isValid()) { context.logErrors(log); - return; + return TaskStatus.error; } ProcessingResults processingResults = new ProcessingResults(expAnnotationIds.size(), log); - processExperiments(_experimentAnnotationsIds, context, processingResults, log); - + return processExperiments(expAnnotationIds, context, processingResults, log) + ? TaskStatus.complete + : TaskStatus.cancelled; } - private void processExperiments(List expAnnotationIds, ProcessingContext context, ProcessingResults processingResults, Logger log) + /** + * @return false if the job was interrupted before every dataset was processed. + */ + private boolean processExperiments(List expAnnotationIds, ProcessingContext context, ProcessingResults processingResults, Logger log) { log.info("Posting reminder message to: {} message threads.", expAnnotationIds.size()); @@ -381,13 +386,15 @@ private void processExperiments(List expAnnotationIds, ProcessingContex { log.info("RUNNING IN TEST MODE - MESSAGES WILL NOT BE POSTED."); } + boolean completed = true; for (Integer experimentAnnotationsId : exptIds) { if (Thread.currentThread().isInterrupted()) { - // Cancelling the job clears the NCBI rate limiter, because every sleep from here on - // throws at once. Stop instead of running the rest at full speed. + // An interrupted thread cannot wait, so the NCBI requests would no longer be spaced. + // The remaining datasets would run back to back. log.warn("Job was interrupted. Stopping before experiment {}.", experimentAnnotationsId); + completed = false; break; } @@ -403,6 +410,7 @@ private void processExperiments(List expAnnotationIds, ProcessingContex } processingResults.logResults(log); + return completed; } private void processExperiment(Integer experimentAnnotationsId, ProcessingContext context, ProcessingResults processingResults) @@ -447,8 +455,7 @@ private void processExperiment(Integer experimentAnnotationsId, ProcessingContex return; } - // Check for publications if enabled. A search that could not run still gets a reminder, since - // the reminder is about the data being private, not about the paper. + // Check for publications if enabled. Send a reminder even if the search fails for any reason. PublicationMatch publicationResult = null; try { @@ -820,7 +827,8 @@ public void logSkipped(Logger log) if (!_publicationSearchFailed.isEmpty()) { - log.error("Publication search failed for the following experiment Ids: {}. NCBI's reason is in the error logged for each one.", StringUtils.join(_publicationSearchFailed, ", ")); + log.error("Publication search failed for {} of {} datasets. Experiment Ids: {}. The NCBI requests that failed are logged as warnings above.", + _publicationSearchFailed.size(), _total, StringUtils.join(_publicationSearchFailed, ", ")); } if (!_submitterNotFound.isEmpty()) diff --git a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp index 4a161bfa..1bdd59f7 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp +++ b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp @@ -99,8 +99,7 @@ const link = document.createElement("a"); link.href = "#"; link.textContent = "Details"; - // The handler is attached here rather than with an onclick attribute, which the Content - // Security Policy blocks. NCBI's reply is encoded because it is third party text. + // NCBI's response is encoded because it is third party text. link.addEventListener("click", function (e) { e.preventDefault(); @@ -119,7 +118,7 @@ LABKEY.Ajax.request({ url: LABKEY.ActionURL.buildURL("panoramapublic", "validateNcbiApiKey.api"), method: "POST", - // An empty value asks the server to check the saved key, which this form never displays. + // An empty value requests a check of the saved key, which this form never displays. jsonData: {ncbiApiKey: input ? input.value : ""}, success: LABKEY.Utils.getCallbackWrapper(function (response) { @@ -228,6 +227,7 @@ " /> <%=button("Validate").onClick("validateNcbiApiKey(); return false;")%> diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java index b0c522db..f051612f 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java @@ -27,8 +27,10 @@ import org.labkey.test.categories.MacCossLabModules; import java.io.IOException; +import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; /** @@ -47,16 +49,21 @@ public class NcbiApiKeyTest extends PanoramaPublicBaseTest private boolean _savedTestApiKey = false; private boolean _useMockNcbi = false; + private Map _originalReminderSettings; @Test public void testNcbiApiKeySettings() { setupMockNcbiService(); + // Capture the existing reminder settings up front so removeTestApiKey can put them back. A dev + // machine may have non-default values set. + _originalReminderSettings = getPrivateDataReminderSettings(); + assertEquals("An NCBI API key is saved on this server. This test saves its own key and cannot" + " restore yours, because a saved key is never readable. Remove the key on the" + " Private Data Reminder Settings page, run this test, then enter the key again.", - "false", getPrivateDataReminderSettings().get("ncbiApiKeySaved")); + "false", _originalReminderSettings.get("ncbiApiKeySaved")); _savedTestApiKey = true; savePrivateDataReminderSettings("2", "0", "0", true, TEST_API_KEY); @@ -81,16 +88,33 @@ public void testNcbiApiKeySettings() } /** - * Covers the button, the request it sends and the message it displays. The mock answers every - * request, so it reports the key as accepted. Only real NCBI rejects one. + * Covers the button, the request it sends and the message it displays. The mock responds to every + * request, so it reports the key as accepted. */ private void verifyValidateButton() { setFormElement(Locator.input("ncbiApiKey"), "not-a-real-key"); click(Locator.lkButton("Validate")); - String expected = _useMockNcbi ? "NCBI accepted this key" : "NCBI rejected this key"; - waitForElement(Locator.id("ncbiApiKeyValidationResult").containing(expected)); + Locator result = Locator.id("ncbiApiKeyValidationResult"); + if (_useMockNcbi) + { + waitForElement(result.containing("NCBI accepted this key")); + } + else + { + // NCBI rejects this key with a 400, or returns a 5xx and the check is unconfirmed. + // Neither reports the key as accepted. The retries mean a live check can take half a minute. + waitFor(() -> { + String text = result.findElement(getDriver()).getText(); + return !text.isEmpty() && !text.startsWith("Checking"); + }, + "NCBI validation result was not displayed", WAIT_FOR_PAGE); + + String message = result.findElement(getDriver()).getText(); + assertFalse("A key NCBI does not recognise must not be reported as accepted. Message: " + message, + message.contains("accepted")); + } setFormElement(Locator.input("ncbiApiKey"), ""); } @@ -149,5 +173,17 @@ public void removeTestApiKey() checkCheckbox(Locator.checkboxByName("clearNcbiApiKey")); clickButton("Save"); } + + if (_originalReminderSettings != null) + { + // The reminder settings are site wide. Restore the values this test overwrote. + savePrivateDataReminderSettings( + _originalReminderSettings.get("extensionLength"), + _originalReminderSettings.get("delayUntilFirstReminder"), + _originalReminderSettings.get("reminderFrequency"), + Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch")), + // A key saved before the run cannot be read back, so leave the key field alone. + null); + } } } diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index 679c51f9..b6f1150a 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -645,9 +645,9 @@ protected Map getPrivateDataReminderSettings() settings.put("reminderFrequency", getFormElement(Locator.input("reminderFrequency"))); settings.put("enablePublicationSearch", String.valueOf(Locator.checkboxByName("enablePublicationSearch").findElement(getDriver()).isSelected())); settings.put("publicationSearchFrequency", getFormElement(Locator.input("publicationSearchFrequency"))); - // The saved key is never displayed. The placeholder is the only signal that one is stored. - settings.put("ncbiApiKeySaved", String.valueOf( - Locator.input("ncbiApiKey").findElement(getDriver()).getDomAttribute("placeholder").startsWith("A key is saved"))); + // The saved key is never displayed. data-key-saved on the field reports whether one is stored. + settings.put("ncbiApiKeySaved", + Locator.input("ncbiApiKey").findElement(getDriver()).getDomAttribute("data-key-saved")); return settings; } diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java index 25124e57..20670d5e 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java @@ -258,10 +258,8 @@ public void testPublicationSearchAndDismiss() assertNotNull("Expected publicationType for dataset 2", dsStatus2AfterPost.get("PublicationType")); assertNotNull("Expected lastReminderDate for dataset 2", dsStatus2AfterPost.get("LastReminderDate")); assertNotNull("Expected citation to be cached for dataset 2", dsStatus2AfterPost.get("Citation")); - } - /* * Navigate to the Panorama Public copy folder and get the experiment ID. */ @@ -495,7 +493,6 @@ public void resetAfterTest() Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch")), null); } - } @Override From f95b5e2c58bf7469c813b7ad8c40a5a2de2817c0 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Mon, 7 Sep 2026 18:47:27 -0700 Subject: [PATCH 14/24] Reported the reminder job's status from what the run recorded, not from log levels * An ERROR through the job's logger sets the pipeline status, and the status set at the end overwrote it. run now computes it from what ProcessingResults recorded * ERROR means the job could not start or a dataset was not sent a reminder. A failed publication search is WARN, since the reminder still goes out * A publication search failing for more than half the searches that reached NCBI is an ERROR * A missing support thread is an error only when the submission has an announcement id. The rest predate the message board and can never be reminded * An unexpected exception on a dataset is now recorded and counted * A citation that will not parse logs at WARN, since it does not fail the match * Added PrivateDataReminderJob.TestCase for the error count, the failure rate boundary, and the announcement id split Co-Authored-By: Claude --- .../panoramapublic/PanoramaPublicModule.java | 2 + .../MockNcbiPublicationSearchService.java | 5 +- .../NcbiPublicationSearchServiceImpl.java | 2 +- .../pipeline/PrivateDataReminderJob.java | 190 ++++++++++++++++-- 4 files changed, 174 insertions(+), 25 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java index aa8e78df..5ece4441 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java @@ -54,6 +54,7 @@ import org.labkey.panoramapublic.model.Journal; import org.labkey.panoramapublic.model.speclib.SpecLibKey; import org.labkey.panoramapublic.pipeline.CopyExperimentPipelineProvider; +import org.labkey.panoramapublic.pipeline.PrivateDataReminderJob; import org.labkey.panoramapublic.pipeline.PxValidationPipelineProvider; import org.labkey.panoramapublic.proteomexchange.ExperimentModificationGetter; import org.labkey.panoramapublic.proteomexchange.Formula; @@ -397,6 +398,7 @@ public Set getSchemaNames() set.add(CatalogEntryManager.TestCase.class); set.add(BlueskyApiClient.TestCase.class); set.add(PrivateDataReminderSettings.TestCase.class); + set.add(PrivateDataReminderJob.TestCase.class); set.add(NcbiPublicationSearchServiceImpl.TestCase.class); set.add(NcbiUtils.TestCase.class); diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java index f5ad1c14..95b02a90 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java @@ -160,8 +160,9 @@ private JSONObject handleESearch(String url) Map> searchMap = isPmc ? _pmcSearchResults : _pubmedSearchResults; // Match search keys against the decoded ESearch query term, so a key cannot match part of - // another parameter such as tool or email. A key is only part of the term - PMC quotes it, - // as in "PXD056793" - so contains() on the term is the right granularity. + // another parameter such as tool or email. A key is only part of the term - searchPmc wraps + // the PMC term in quotes, as in "PXD056793" - so contains() on the term is the right + // granularity. String term = extractQueryParam(url, "term"); JSONArray idList = new JSONArray(); diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index 62cc209c..e47e827a 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -208,7 +208,7 @@ static String parseCitation(String response, String publicationId, DB database, } catch (JSONException e) { - log.error("Error parsing response from NCBI Literature Citation Exporter for {} ID {}", database.getLabel(), publicationId, e); + log.warn("Error parsing response from NCBI Literature Citation Exporter for {} ID {}", database.getLabel(), publicationId, e); } return null; } diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index 6d4d7b80..a2f07a4c 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -19,6 +19,8 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; import org.labkey.api.announcements.api.Announcement; import org.labkey.api.announcements.api.AnnouncementService; import org.labkey.api.data.Container; @@ -32,6 +34,7 @@ import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.URLHelper; +import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.ViewBackgroundInfo; import org.labkey.panoramapublic.PanoramaPublicManager; import org.labkey.panoramapublic.PanoramaPublicNotification; @@ -39,6 +42,7 @@ import org.labkey.panoramapublic.model.DatasetStatus; import org.labkey.panoramapublic.model.ExperimentAnnotations; import org.labkey.panoramapublic.model.Journal; +import org.labkey.panoramapublic.model.JournalExperiment; import org.labkey.panoramapublic.model.JournalSubmission; import org.labkey.panoramapublic.ncbi.NcbiApiKeyCheck; import org.labkey.panoramapublic.ncbi.NcbiPublicationSearchService; @@ -61,6 +65,20 @@ public class PrivateDataReminderJob extends PipelineJob { + // Below this many datasets the failure rate says too little to act on. + private static final int MIN_DATASETS_FOR_FAILURE = 3; + private static final double PUBLICATION_SEARCH_FAILURE_THRESHOLD = 0.5; + + /** + * @return true when the publication search failure rate exceeds the threshold, which suggests a + * problem with searching NCBI rather than with one dataset. + */ + static boolean publicationSearchFailingWidely(int failed, int attempted) + { + return attempted >= MIN_DATASETS_FOR_FAILURE + && failed > attempted * PUBLICATION_SEARCH_FAILURE_THRESHOLD; + } + private boolean _test; private boolean _forcePublicationCheck; private List _experimentAnnotationsIds; @@ -189,22 +207,24 @@ public static ReminderDecision skip(String reason) { public boolean shouldPost() { return shouldPost; } public String getReason() { return reason; } } - + /** - * Checks for publications associated with the experiment if enabled in settings - * @param expAnnotations The experiment to check - * @param settings The reminder settings - * @param forceCheck Force publication check regardless of global setting - * @param log Logger for diagnostic messages - * @return NcbiArticleMatch with search result + * Finds the publication to report for the experiment, from the dataset's cached match or by + * searching NCBI. + * + * @return null when there is nothing new to report, which includes a search that found only the + * publication the submitter has already dismissed + * @throws NcbiSearchException if a search ran and could not complete */ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations expAnnotations, @NotNull PrivateDataReminderSettings settings, boolean forceCheck, @NotNull User user, boolean testMode, - @NotNull Logger log) + @NotNull ProcessingResults results) { + Logger log = results._log; + // Check if publication checking is enabled (either globally or forced for this run) if (!forceCheck && !settings.isEnablePublicationSearch()) { @@ -230,6 +250,7 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp log.info("Search deferral expired for experiment {} (dismissed {}); re-searching NCBI", expAnnotations.getId(), dismissedDate); try { + results.addPublicationSearchAttempted(); PublicationMatch newMatch = NcbiPublicationSearchService.get().searchForPublication(expAnnotations, log); if (newMatch != null && !newMatch.getPublicationId().equals(datasetStatus.getPotentialPublicationId())) { @@ -260,7 +281,8 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp } catch (Exception e) { - log.error("Error re-searching publication for experiment {}: {}", expAnnotations.getId(), e.getMessage(), e); + // The reminder is still posted, so this costs the paper information and nothing else. + log.warn("Error re-searching publication for experiment {}: {}", expAnnotations.getId(), e.getMessage(), e); return null; } } @@ -277,6 +299,7 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp log.info("Searching for publications for experiment {}", expAnnotations.getId()); try { + results.addPublicationSearchAttempted(); return NcbiPublicationSearchService.get().searchForPublication(expAnnotations, log); } catch (NcbiSearchException e) @@ -286,7 +309,8 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp } catch (Exception e) { - log.error("Error searching for publication for experiment {}: {}", expAnnotations.getId(), e.getMessage(), e); + // The reminder is still posted, so this costs the paper information and nothing else. + log.warn("Error searching for publication for experiment {}: {}", expAnnotations.getId(), e.getMessage(), e); return null; } } @@ -348,8 +372,9 @@ private boolean ncbiApiKeyAccepted() } /** - * @return complete when every dataset was processed, error when the job could not start, and - * cancelled when it was interrupted partway. + * @return error when the job could not start, a dataset that should have been sent a reminder was + * not, or the publication search failed for most of the datasets it ran for. Cancelled when the + * job was interrupted with nothing else to report, and complete otherwise. */ private TaskStatus postMessage(List expAnnotationIds, Journal panoramaPublic) { @@ -369,9 +394,15 @@ private TaskStatus postMessage(List expAnnotationIds, Journal panoramaP } ProcessingResults processingResults = new ProcessingResults(expAnnotationIds.size(), log); - return processExperiments(expAnnotationIds, context, processingResults, log) - ? TaskStatus.complete - : TaskStatus.cancelled; + boolean completed = processExperiments(expAnnotationIds, context, processingResults, log); + + // An ERROR logged through the job's logger sets the status to error, and the status set here + // would overwrite it. Report the errors the run recorded instead. + if (processingResults.getTotalErrors() > 0 || processingResults.publicationSearchFailingWidely()) + { + return TaskStatus.error; + } + return completed ? TaskStatus.complete : TaskStatus.cancelled; } /** @@ -405,7 +436,7 @@ private boolean processExperiments(List expAnnotationIds, ProcessingCon } catch (Exception e) { - log.error("Error processing experiment {}: {}", experimentAnnotationsId, e.getMessage(), e); + processingResults.addProcessingFailed(experimentAnnotationsId, e); } } @@ -459,7 +490,7 @@ private void processExperiment(Integer experimentAnnotationsId, ProcessingContex PublicationMatch publicationResult = null; try { - publicationResult = searchForPublication(expAnnotations, context.getSettings(), _forcePublicationCheck, getUser(), context.isTestMode(), processingResults._log); + publicationResult = searchForPublication(expAnnotations, context.getSettings(), _forcePublicationCheck, getUser(), context.isTestMode(), processingResults); } catch (NcbiSearchException e) { @@ -740,8 +771,11 @@ private static class ProcessingResults private final List _experimentNotFound = new ArrayList<>(); private final List _submissionNotFound = new ArrayList<>(); private final List _announcementNotFound = new ArrayList<>(); + private final List _noSupportThread = new ArrayList<>(); private final List _submitterNotFound = new ArrayList<>(); private final List _publicationSearchFailed = new ArrayList<>(); + private int _publicationSearchAttempted = 0; + private final List _processingFailed = new ArrayList<>(); private final List _skipped = new ArrayList<>(); private int _processed = 0; private final int _total; @@ -772,6 +806,13 @@ public void addLatestSubmissionNotFound(Integer experimentId) public void addAnnouncementNotFound(Integer experimentId, JournalSubmission submission, Container announcementsFolder) { + if (submission.getAnnouncementId() == null) + { + // Data submitted before Panorama Public started posting submission requests to a + // message board. + _noSupportThread.add(experimentId); + return; + } _announcementNotFound.add(experimentId); _log.error("Could not find the message thread for experiment Id: {}; announcement Id: {} in the folder {}.", experimentId, submission.getAnnouncementId(), announcementsFolder.getPath()); } @@ -782,10 +823,21 @@ public void addSubmitterNotFound(Integer experimentId) _log.error("Could not find a submitter user for experiment Id: {}.", experimentId); } + public void addPublicationSearchAttempted() + { + _publicationSearchAttempted++; + } + public void addPublicationSearchFailed(Integer experimentId, Exception e) { _publicationSearchFailed.add(experimentId); - _log.error("Publication search failed for experiment Id: {}. A reminder was still posted. {}", experimentId, e.getMessage(), e); + _log.warn("Publication search failed for experiment Id: {}. A reminder was still posted. {}", experimentId, e.getMessage(), e); + } + + public void addProcessingFailed(Integer experimentId, Exception e) + { + _processingFailed.add(experimentId); + _log.error("Error processing experiment {}: {}", experimentId, e.getMessage(), e); } public void addSkipped(Integer experimentId, ReminderDecision decision) @@ -825,10 +877,28 @@ public void logSkipped(Logger log) log.error("Support message threads were not found for the following experiment Ids: {}", StringUtils.join(_announcementNotFound, ", ")); } + if (!_noSupportThread.isEmpty()) + { + log.warn("The following experiment Ids were submitted before Panorama Public posted submission requests to a message board, so they have no support message thread and cannot be sent a reminder: {}", + StringUtils.join(_noSupportThread, ", ")); + } + if (!_publicationSearchFailed.isEmpty()) { - log.error("Publication search failed for {} of {} datasets. Experiment Ids: {}. The NCBI requests that failed are logged as warnings above.", - _publicationSearchFailed.size(), _total, StringUtils.join(_publicationSearchFailed, ", ")); + String message = "Publication search failed for {} of the {} datasets a search was run for. Experiment Ids: {}. The NCBI requests that failed are logged as warnings above."; + if (publicationSearchFailingWidely()) + { + log.error(message, _publicationSearchFailed.size(), _publicationSearchAttempted, StringUtils.join(_publicationSearchFailed, ", ")); + } + else + { + log.warn(message, _publicationSearchFailed.size(), _publicationSearchAttempted, StringUtils.join(_publicationSearchFailed, ", ")); + } + } + + if (!_processingFailed.isEmpty()) + { + log.error("Processing failed for the following experiment Ids: {}", StringUtils.join(_processingFailed, ", ")); } if (!_submitterNotFound.isEmpty()) @@ -842,12 +912,23 @@ public void logSkipped(Logger log) } } + public boolean publicationSearchFailingWidely() + { + return PrivateDataReminderJob.publicationSearchFailingWidely( + _publicationSearchFailed.size(), _publicationSearchAttempted); + } + + /** + * @return the number of datasets that should have been sent a reminder and were not. Datasets + * whose publication search failed are not counted, because the reminder was still posted. + */ public int getTotalErrors() { return _experimentNotFound.size() + _submissionNotFound.size() + _announcementNotFound.size() + - _submitterNotFound.size(); + _submitterNotFound.size() + + _processingFailed.size(); } public void logSummary(Logger log) { @@ -861,7 +942,72 @@ public void logSummary(Logger log) log.info("Successfully processed {}.", StringUtilsLabKey.pluralize(_processed, "experiment")); } - log.info("Processing complete: {} total, {} processed, {} skipped, {} errors", _total, _processed, _skipped.size(), getTotalErrors()); + log.info("Processing complete: {} total, {} processed, {} skipped, {} with no support message thread, {} errors", + _total, _processed, _skipped.size(), _noSupportThread.size(), getTotalErrors()); + } + } + + public static class TestCase extends Assert + { + private static final Logger TEST_LOG = LogHelper.getLogger(TestCase.class, "Private data reminder job tests"); + + @Test + public void testPublicationSearchFailingWidely() + { + // Below the floor the rate says too little to act on, even when every search failed. + assertFalse("2 of 2 is under the floor", publicationSearchFailingWidely(2, 2)); + + // At the floor, more than half the searches that ran. + assertTrue("2 of 3 is over half", publicationSearchFailingWidely(2, 3)); + assertFalse("1 of 3 is not over half", publicationSearchFailingWidely(1, 3)); + + // Exactly half is not enough. The comparison is strict. + assertFalse("2 of 4 is exactly half", publicationSearchFailingWidely(2, 4)); + assertTrue("3 of 4 is over half", publicationSearchFailingWidely(3, 4)); + + assertFalse("No failures", publicationSearchFailingWidely(0, 10)); + assertFalse("No searches ran", publicationSearchFailingWidely(0, 0)); + } + + @Test + public void testGetTotalErrors() + { + ProcessingResults results = new ProcessingResults(10, TEST_LOG); + assertEquals("A run with nothing recorded has no errors", 0, results.getTotalErrors()); + + results.addExperimentNotFound(1); + results.addSubmissionNotFound(2); + results.addLatestSubmissionNotFound(3); + results.addSubmitterNotFound(4); + results.addProcessingFailed(5, new IllegalStateException("test")); + assertEquals("Each dataset that missed its reminder is counted once", 5, results.getTotalErrors()); + + // The reminder was still posted for these, so they are not errors. + results.addPublicationSearchFailed(6, new NcbiSearchException("test")); + assertEquals("A failed publication search is not a missed reminder", 5, results.getTotalErrors()); + } + + @Test + public void testAnnouncementNotFoundSplitsOnAnnouncementId() + { + ProcessingResults results = new ProcessingResults(10, TEST_LOG); + Container container = ContainerManager.getRoot(); + + // No announcement id means the data predates the support message board. It cannot be sent + // a reminder and is not an error. + results.addAnnouncementNotFound(1, submissionWithAnnouncementId(null), container); + assertEquals("A dataset that never had a thread is not an error", 0, results.getTotalErrors()); + + // An announcement id whose thread could not be read is worth investigating. + results.addAnnouncementNotFound(2, submissionWithAnnouncementId(99), container); + assertEquals("A thread that went missing is an error", 1, results.getTotalErrors()); + } + + private static JournalSubmission submissionWithAnnouncementId(Integer announcementId) + { + JournalExperiment journalExperiment = new JournalExperiment(); + journalExperiment.setAnnouncementId(announcementId); + return new JournalSubmission(journalExperiment); } } } From 77d8727724a7f565b25a27d471bce28ad2feed62 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 27 Sep 2026 09:47:34 -0700 Subject: [PATCH 15/24] Fixed the NCBI API key settings form and retried connection resets * PrivateDataReminderSettingsAction rejects a new key entered with "Remove the saved key" checked. The typed key used to be discarded while the page reported success * The settings form reports a saved key after a validation error, not only on first display * PrivateDataReminderSettings.get reads the key whether or not other reminder settings are saved * NcbiPublicationSearchServiceImpl.isRetryable retries a SocketException (connection reset) * publicationSearchFailingWidely triggers at half or more of the searches, not more than half * Renamed the PrivateDataReminderSettings.TestCase boundary helpers and dropped an unused argument * Reworded comments in the NCBI classes, the reminder job and the Selenium tests. US spelling Co-Authored-By: Claude --- .../PanoramaPublicController.java | 18 +++-- .../panoramapublic/PanoramaPublicModule.java | 3 +- .../message/PrivateDataReminderSettings.java | 77 +++++++++---------- .../MockNcbiPublicationSearchService.java | 7 +- .../panoramapublic/ncbi/NcbiApiKeyCheck.java | 4 +- .../NcbiPublicationSearchServiceImpl.java | 65 +++++++--------- .../ncbi/NcbiSearchException.java | 4 +- .../pipeline/PrivateDataReminderJob.java | 21 +++-- .../tests/panoramapublic/NcbiApiKeyTest.java | 3 +- .../PanoramaPublicBaseTest.java | 4 +- .../panoramapublic/PublicationSearchTest.java | 2 - 11 files changed, 94 insertions(+), 114 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java index d5c6b32f..346b59d8 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java @@ -10094,8 +10094,7 @@ public Object execute(PrivateDataReminderSettingsForm form, BindException errors ApiSimpleResponse response = new ApiSimpleResponse(); response.put("success", true); - // An empty field means check the key that is already saved, since the form never - // displays it. + // An empty field means check the key that is already saved. boolean checkingSavedKey = StringUtils.isBlank(form.getNcbiApiKey()); String apiKey = checkingSavedKey ? PrivateDataReminderSettings.get().getNcbiApiKey() @@ -10112,8 +10111,7 @@ public Object execute(PrivateDataReminderSettingsForm form, BindException errors response.put("valid", check.isValid()); if (check.isValid()) { - // Validating does not store anything, so say so. Otherwise "accepted" reads as - // confirmation that the key is now in effect. + // Validating does not save the key. Inform the user that the key needs to be saved. response.put("message", checkingSavedKey ? "NCBI accepted the saved key." : "NCBI accepted this key. Click Save to store it."); @@ -10121,8 +10119,7 @@ public Object execute(PrivateDataReminderSettingsForm form, BindException errors } else { - // The short message goes beside the field. NCBI's own words are offered separately, - // since the admin holding the key is the one who has to act on them. + // The short message is displayed next to the field. NCBI's own message is displayed separately. response.put("message", check.isRejected() ? "NCBI rejected this key." : "Could not reach NCBI to check this key."); @@ -10181,6 +10178,10 @@ else if (PrivateDataReminderSettings.parseReminderTime(form.getReminderTime()) = errors.reject(ERROR_MSG, String.format("'Reminder time' could not be parsed. It must be in the format - %s, e.g. %s.", PrivateDataReminderSettings.REMINDER_TIME_FORMAT, PrivateDataReminderSettings.DEFAULT_REMINDER_TIME)); } + if (form.isClearNcbiApiKey() && !StringUtils.isBlank(form.getNcbiApiKey())) + { + errors.reject(ERROR_MSG, "Enter a new NCBI API key or select 'Remove the saved key', not both."); + } } @Override @@ -10196,10 +10197,11 @@ public ModelAndView getView(PrivateDataReminderSettingsForm form, boolean reshow form.setExtensionLength(settings.getExtensionLength()); form.setEnablePublicationSearch(settings.isEnablePublicationSearch()); form.setPublicationSearchFrequency(settings.getPublicationSearchFrequency()); - // Do not put the saved key in the form. The JSP shows only whether one is stored. - form.setNcbiApiKeySet(PrivateDataReminderSettings.hasNcbiApiKey()); } + // Rendered as an attribute, not an input, so the form does not post it back. Set on every render. + form.setNcbiApiKeySet(PrivateDataReminderSettings.hasNcbiApiKey()); + VBox view = new VBox(); view.addView(new JspView<>("/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp", form, errors)); view.setTitle("Private Data Reminder Settings"); diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java index 5ece4441..791d165a 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java @@ -162,8 +162,7 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) @Override public void startBackgroundThreads() { - // Re-establish the reminder schedule on every startup. Reminder messages contain absolute - // URLs, which are only safe to build once this method is called. + // Re-establish the reminder schedule on every startup. try { PrivateDataMessageScheduler.getInstance().initialize(PrivateDataReminderSettings.get().isEnableReminders()); diff --git a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java index e1439331..90c360fc 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java +++ b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java @@ -106,8 +106,6 @@ public static PrivateDataReminderSettings get() ? DEFAULT_PUBLICATION_SEARCH_FREQUENCY : Integer.valueOf(settingsMap.get(PROP_PUBLICATION_SEARCH_FREQUENCY)); settings.setPublicationSearchFrequency(publicationSearchFrequency); - - settings.setNcbiApiKey(getNcbiApiKeyValue()); } else { @@ -120,6 +118,9 @@ public static PrivateDataReminderSettings get() settings.setPublicationSearchFrequency(DEFAULT_PUBLICATION_SEARCH_FREQUENCY); } + // Read the key from its own property set. + settings.setNcbiApiKey(getNcbiApiKeyValue()); + return settings; } @@ -158,8 +159,7 @@ public static void save(PrivateDataReminderSettings settings) } /** - * Save the NCBI API key, or remove it when the key is blank. The key lives in the encrypted - * store, like the other credentials this module holds. + * Save the NCBI API key, or remove it when the key is blank. The key lives in the encrypted store. */ public static void saveNcbiApiKey(@Nullable String apiKey) { @@ -414,26 +414,30 @@ private void testExtensionIsExpired(PrivateDataReminderSettings settings, int mo testExtensionIsValid(settings, monthsOffset, false); } - private void testExtensionIsValidAsOf(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset) + private void testExtensionIsValidAsOf(PrivateDataReminderSettings settings, int monthsOffset, + int minutesBeforeExpiry) + { + testExtensionNearExpiry(settings, monthsOffset, minutesBeforeExpiry, true); + } + + private void testExtensionIsExpiredAsOf(PrivateDataReminderSettings settings, int monthsOffset, + int minutesBeforeExpiry) { - testExtensionAtExpiry(settings, monthsOffset, minutesOffset, true); + testExtensionNearExpiry(settings, monthsOffset, minutesBeforeExpiry, false); } /** - * Checks the boundary at the moment an extension expires. The current date comes from the - * expiry the settings calculate, not from today. Subtracting months and adding them back does - * not always return to the same day, because a shorter target month clamps the day, so a date - * built from today can sit on the wrong side of the boundary. On 31 August, subtracting six - * months gives 28 February and adding six back gives 28 August. + * Checks the boundary at the moment an extension expires. The current date comes from the expiry + * the settings calculate, because subtracting months and adding them back does not always return + * to the same day. From 31 August, six months back is 28 February, and six months on is 28 August. * - * @param minutesBeforeExpiry how long before the expiry to check. Zero is the expiry itself, - * and a negative value is after it. + * @param minutesBeforeExpiry zero is the expiry itself, and a negative value is after it. */ - private void testExtensionAtExpiry(PrivateDataReminderSettings settings, int monthsOffset, - int minutesBeforeExpiry, boolean expectedValid) + private void testExtensionNearExpiry(PrivateDataReminderSettings settings, int monthsOffset, + int minutesBeforeExpiry, boolean expectedValid) { DatasetStatus datasetStatus = new DatasetStatus(); - datasetStatus.setExtensionRequestedDate(dateFromNow(monthsOffset, 0, 0)); + datasetStatus.setExtensionRequestedDate(dateFromNow(monthsOffset, 0)); Date expiry = settings.getExtensionValidUntilDate(datasetStatus); Date currentDate = Date.from(expiry.toInstant().minusSeconds(minutesBeforeExpiry * 60L)); @@ -448,15 +452,10 @@ private void testExtensionAtExpiry(PrivateDataReminderSettings settings, int mon else assertFalse(failureMessage, isValid); } - private void testExtensionIsExpiredAsOf(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset) - { - testExtensionAtExpiry(settings, monthsOffset, minutesOffset, false); - } - private void testExtensionIsValid(PrivateDataReminderSettings settings, int monthsOffset, boolean expectedValid) { DatasetStatus datasetStatus = new DatasetStatus(); - datasetStatus.setExtensionRequestedDate(dateFromNow(monthsOffset, 0, 0)); + datasetStatus.setExtensionRequestedDate(dateFromNow(monthsOffset, 0)); String failureMessage = String.format("Extension is %s; Extension Length: %d; Extension Requested On: %s; Valid Until: %s", expectedValid ? "valid" : "expired", @@ -479,29 +478,29 @@ private void testReminderIsOld(PrivateDataReminderSettings settings, int daysOff testReminderIsRecent(settings, daysOffset, false); } - private void testReminderIsRecentAsOf(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset) + private void testReminderIsRecentAsOf(PrivateDataReminderSettings settings, int monthsOffset, + int minutesBeforeExpiry) { - testReminderAtExpiry(settings, monthsOffset, minutesOffset, true); + testReminderNearExpiry(settings, monthsOffset, minutesBeforeExpiry, true); } - private void testReminderIsOldAsOf(PrivateDataReminderSettings settings, int monthsOffset, int minutesOffset) + private void testReminderIsOldAsOf(PrivateDataReminderSettings settings, int monthsOffset, + int minutesBeforeExpiry) { - testReminderAtExpiry(settings, monthsOffset, minutesOffset, false); + testReminderNearExpiry(settings, monthsOffset, minutesBeforeExpiry, false); } /** - * Checks the boundary at the moment a reminder stops counting as recent. The current date - * comes from the date the settings calculate, for the reason given on - * {@link #testExtensionAtExpiry}. + * Checks the boundary at the moment a reminder stops counting as recent. The current date comes + * from the expiry, as {@link #testExtensionNearExpiry} explains. * - * @param minutesBeforeExpiry how long before that date to check. Zero is the date itself, and - * a negative value is after it. + * @param minutesBeforeExpiry zero is the expiry itself, and a negative value is after it. */ - private void testReminderAtExpiry(PrivateDataReminderSettings settings, int monthsOffset, - int minutesBeforeExpiry, boolean expectedRecent) + private void testReminderNearExpiry(PrivateDataReminderSettings settings, int monthsOffset, + int minutesBeforeExpiry, boolean expectedRecent) { DatasetStatus datasetStatus = new DatasetStatus(); - datasetStatus.setLastReminderDate(dateFromNow(monthsOffset, 0, 0)); + datasetStatus.setLastReminderDate(dateFromNow(monthsOffset, 0)); Date expiry = settings.getReminderValidUntilDate(datasetStatus); Date currentDate = Date.from(expiry.toInstant().minusSeconds(minutesBeforeExpiry * 60L)); @@ -519,7 +518,7 @@ private void testReminderAtExpiry(PrivateDataReminderSettings settings, int mont private void testReminderIsRecent(PrivateDataReminderSettings settings, int daysOffset, boolean expectedRecent) { DatasetStatus datasetStatus = new DatasetStatus(); - datasetStatus.setLastReminderDate(dateFromNow(0, daysOffset, 0)); + datasetStatus.setLastReminderDate(dateFromNow(0, daysOffset)); String failureMessage = String.format("Reminder is %s; Reminder Frequency: %d; Reminder Sent On: %s; Valid Until: %s", expectedRecent ? "recent" : "old", @@ -550,19 +549,13 @@ private PrivateDataReminderSettings createTestSettings(int extensionLength, int return testSettings; } - private Date dateFromNow() - { - return dateFromNow(0, 0, 0); - } - - private Date dateFromNow(int monthsOffset, int daysOffset, int minutesOffset) + private Date dateFromNow(int monthsOffset, int daysOffset) { return Date.from( LocalDate.now() .plusMonths(monthsOffset) .plusDays(daysOffset) .atStartOfDay(ZoneId.systemDefault()) - .plusMinutes(minutesOffset) .toInstant() ); } diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java index 95b02a90..af26e980 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java @@ -160,9 +160,7 @@ private JSONObject handleESearch(String url) Map> searchMap = isPmc ? _pmcSearchResults : _pubmedSearchResults; // Match search keys against the decoded ESearch query term, so a key cannot match part of - // another parameter such as tool or email. A key is only part of the term - searchPmc wraps - // the PMC term in quotes, as in "PXD056793" - so contains() on the term is the right - // granularity. + // another parameter such as tool or email. String term = extractQueryParam(url, "term"); JSONArray idList = new JSONArray(); @@ -170,6 +168,7 @@ private JSONObject handleESearch(String url) { for (Map.Entry> entry : searchMap.entrySet()) { + // A key is a substring of the term. searchPmc wraps the PMC term in quotes, e.g. "PXD056793". if (term.contains(entry.getKey())) { entry.getValue().forEach(idList::put); @@ -188,7 +187,7 @@ private JSONObject handleESummary(String url) Map metadataMap = isPmc ? _pmcMetadata : _pubmedMetadata; // ESummary requests a comma-separated list of IDs in the "id" parameter. Match registered - // IDs against that list (exactly, not by substring), rather than scanning the whole URL. + // IDs against that list rather than scanning the whole URL. String idParam = extractQueryParam(url, "id"); List requestedIds = idParam == null ? List.of() : Arrays.asList(idParam.split(",")); diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiApiKeyCheck.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiApiKeyCheck.java index b7bcf95e..5c4a9ecf 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiApiKeyCheck.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiApiKeyCheck.java @@ -18,8 +18,8 @@ import org.jetbrains.annotations.Nullable; /** - * The outcome of checking an NCBI API key. A rejected key is a configuration problem an admin has to - * correct. A check that could not be completed is not, so callers need to tell the two apart. + * The outcome of checking an NCBI API key, which means sending NCBI a request that carries the key + * and reading the status it returns. */ public class NcbiApiKeyCheck { diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index e47e827a..27c46550 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -44,6 +44,7 @@ import org.labkey.panoramapublic.ncbi.NcbiConstants.DB; import java.io.IOException; +import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -105,7 +106,7 @@ public static void setInstance(NcbiPublicationSearchService impl) private static final int MAX_HTTP_ATTEMPTS = 3; // initial try + 2 retries private static final int RETRY_BASE_DELAY_MS = 500; // exponential backoff base - private static final int BAD_REQUEST = 400; // NCBI's response when the API key is not recognised + private static final int BAD_REQUEST = 400; // NCBI's response when the API key is not recognized private static final int TOO_MANY_REQUESTS = 429; // NCBI's response when the request rate is exceeded private static final int MAX_ERROR_BODY_CHARS = 500; @@ -178,8 +179,7 @@ private static Logger getLog(@Nullable Logger logger) } catch (IOException e) { - // A missing citation does not fail the match. getCitation returns null and the caller - // displays the publication ID instead. + // A missing citation does not fail the match. The caller displays the publication ID instead. log.warn("Request to the NCBI Literature Citation Exporter did not complete. URL: {}", queryUrl, e); } return null; @@ -228,7 +228,7 @@ public List searchForPublication(@NotNull ExperimentAnnotation log.info("Starting publication search for experiment: {}", expAnnotations.getId()); // NCBI requests that failed after retries. An empty result is returned only when every request - // completed, so a caller can tell a search that found no publication from one that could not run. + // completed, so a caller can distinguish a search that found no publication from one that failed to run. List failedRequests = new ArrayList<>(); // Search PubMed Central first @@ -304,8 +304,8 @@ public List searchForPublication(@NotNull ExperimentAnnotation } catch (NcbiSearchException e) { - // NCBI responds inconsistently to the same query, so the remaining strategies are - // still worth running. executeSearch has already logged the query and the cause. + // Each strategy is a separate request, so the remaining ones are still worth + // running. Not logged here, executeSearch logs the query and the cause. failedRequests.add("PMC search by " + strategy); } rateLimit(); @@ -393,8 +393,7 @@ private List executeSearch(String query, String database, Logger log) } catch (IOException | JSONException e) { - // One request, not the outcome for the dataset. The search methods catch this per request - // and carry on with the remaining ones. + // One failed request, not the outcome for the dataset. The caller carries on with the rest. log.warn("Search of {} did not complete for query: {}", database, query, e); throw new NcbiSearchException("Error searching " + database + " with query: " + query, e); } @@ -403,8 +402,7 @@ private List executeSearch(String query, String database, Logger log) @Override public @NotNull NcbiApiKeyCheck checkApiKey(@Nullable String apiKey) { - // A minimal ESearch request. A rejected key comes back as a 400 on the first attempt, while - // a 5xx or 429 is retried like any other request before it is called unconfirmed. + // A minimal ESearch request. getString retries a transient failure before the check gives up. String url = ESEARCH_URL + "?" + buildCommonParams("pubmed", apiKey) + "&term=labkey&retmax=1&retmode=json"; try { @@ -413,8 +411,8 @@ private List executeSearch(String query, String database, Logger log) } catch (HttpResponseException e) { - // NCBI rejects a key it does not recognise with a 400. Any other status says nothing about - // the key, including a 403 or 404 from a proxy between the server and NCBI. + // NCBI rejects an unrecognized key with a 400. Any other status says nothing about the + // key, including a 403 or 404 from a proxy between the server and NCBI. String message = redactApiKey(e.getMessage(), apiKey); return e.getStatusCode() == BAD_REQUEST ? NcbiApiKeyCheck.rejected(message) @@ -438,8 +436,7 @@ protected JSONObject getJson(String url, Logger log) throws IOException /** * Execute an HTTP GET request and return the response body as a string. Retry warnings are - * written to {@code log}. The reminder job passes its pipeline job logger, and UI-triggered - * searches pass the static server logger. + * written to {@code log}. * @throws IOException if the request fails or the server returns a non-2xx response */ protected String getString(String url, Logger log) throws IOException @@ -487,8 +484,8 @@ protected String executeGet(String url) throws IOException try (CloseableHttpClient client = HttpClientBuilder.create() .setDefaultRequestConfig(requestConfig) .setConnectionManager(connectionManager) - // HttpClient retries 429 and 503 once on its own, which would make MAX_HTTP_ATTEMPTS - // cost twice the requests it names. getString is the only retry. + // Without this, the number of requests would be up to twice MAX_HTTP_ATTEMPTS. isRetryable + // has to cover what this turns off. .disableAutomaticRetries() .build()) { @@ -507,10 +504,9 @@ protected String executeGet(String url) throws IOException } /** - * Read the body of a client error response. 5xx bodies are large, uninformative HTML error - * pages, so only client error bodies are read, and only the first {@link #MAX_ERROR_BODY_CHARS} - * characters. A body that cannot be read or parsed returns null, because losing NCBI's text is - * better than losing the status code the caller decides on. + * Read the body of a client error response, up to {@link #MAX_ERROR_BODY_CHARS} characters. + * 5xx bodies are large, uninformative HTML error pages, so they are skipped. Returns null if + * the body cannot be read. */ private static @Nullable String readErrorBody(int status, ClassicHttpResponse response) { @@ -530,10 +526,9 @@ protected String executeGet(String url) throws IOException } /** - * Build the message for a non-2xx HttpResponseException. For client errors (4xx) the response - * body is appended, so NCBI's reason (e.g. "API key invalid") reaches the log. Other statuses - * use only the reason phrase. The body is third-party text on its way to a log, so any - * occurrence of the API key is removed from it. + * Build the message for a non-2xx HttpResponseException. A 4xx appends the response body, so + * NCBI's reason (e.g. "API key invalid") reaches the log. Other statuses use only the reason + * phrase. The API key is removed from the body, which is third-party text bound for a log. */ static String errorDetail(int status, String reasonPhrase, @Nullable String body, @Nullable String apiKey) { @@ -545,9 +540,7 @@ static String errorDetail(int status, String reasonPhrase, @Nullable String body } /** - * Replace the NCBI API key wherever it appears in text that is about to be logged. The key is a - * query parameter on every eutils URL. When the reminder job runs, the log is the pipeline job - * log, which is readable by anyone with read access to the folder the job ran in. + * Replace the NCBI API key wherever it appears in text bound for a log. */ static @Nullable String redactApiKey(@Nullable String text, @Nullable String apiKey) { @@ -573,13 +566,12 @@ static String errorDetail(int status, String reasonPhrase, @Nullable String body } /** - * Read timeouts, 5xx responses and 429 are transient NCBI failures worth retrying. NCBI returns - * 429 when the request rate is exceeded, which is the failure backoff exists for. Other 4xx - * errors are permanent. + * Read timeouts, connection resets, 5xx responses and 429 are transient NCBI failures worth + * retrying. Other 4xx errors are permanent. */ private static boolean isRetryable(IOException e) { - if (e instanceof SocketTimeoutException) + if (e instanceof SocketTimeoutException || e instanceof SocketException) { return true; } @@ -590,8 +582,7 @@ private static boolean isRetryable(IOException e) return false; } - // 500ms after the first failure, then doubling. No jitter. Each caller issues its NCBI requests - // sequentially. Retries collide only when the reminder job and a UI search overlap, which is rare. + // 500ms after the first failure, then doubling. private static long retryDelayMs(int attempt) { return (long) RETRY_BASE_DELAY_MS << (attempt - 1); @@ -855,8 +846,7 @@ private List searchPubMed(ExperimentAnnotations expAnnotations } catch (NcbiSearchException e) { - // Author and title cannot be verified without the metadata, and an unverified PMID is not - // a match. + // Metadata is required to confirm that a PMID is a match. failedRequests.add("PubMed metadata fetch"); return Collections.emptyList(); } @@ -1198,8 +1188,7 @@ private static String buildCommonParams(String database, @Nullable String apiKey "&tool=" + URLEncoder.encode(TOOL, StandardCharsets.UTF_8) + "&email=" + URLEncoder.encode(EMAIL, StandardCharsets.UTF_8); - // An NCBI API key (configured in the Private Data Reminder Settings) raises the eutils - // rate limit from 3 to 10 requests/sec. + // An API key raises the eutils rate limit from 3 to 10 requests/sec. if (!StringUtils.isBlank(apiKey)) { params += "&api_key=" + URLEncoder.encode(apiKey.trim(), StandardCharsets.UTF_8); @@ -1809,7 +1798,7 @@ protected boolean sleepMs(long ms) @Test public void testCheckApiKey() { - // NCBI rejects a key it does not recognise with a 400. The reminder job stops for that, + // NCBI rejects a key it does not recognize with a 400. The reminder job stops for that, // so a 5xx has to be reported as a check that could not be completed. assertEquals(NcbiApiKeyCheck.Status.REJECTED, checkApiKeyAgainst(new HttpResponseException(400, "Bad Request")).getStatus()); assertEquals(NcbiApiKeyCheck.Status.UNCONFIRMED, checkApiKeyAgainst(new HttpResponseException(503, "Service Unavailable")).getStatus()); diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java index 907cd0b1..4ff46cb9 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java @@ -16,8 +16,8 @@ package org.labkey.panoramapublic.ncbi; /** - * Lets a caller tell a failed NCBI request from a search that found nothing. Without it a rejected - * API key and a dataset with no published paper both arrive as an empty result. + * Lets a caller distinguish a failed NCBI request from a search that found nothing. + * searchForPublication throws it when no publication was found and at least one request failed. */ public class NcbiSearchException extends RuntimeException { diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index a2f07a4c..86509887 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -65,18 +65,17 @@ public class PrivateDataReminderJob extends PipelineJob { - // Below this many datasets the failure rate says too little to act on. private static final int MIN_DATASETS_FOR_FAILURE = 3; private static final double PUBLICATION_SEARCH_FAILURE_THRESHOLD = 0.5; /** - * @return true when the publication search failure rate exceeds the threshold, which suggests a + * @return true when the publication search failure rate reaches the threshold, which suggests a * problem with searching NCBI rather than with one dataset. */ static boolean publicationSearchFailingWidely(int failed, int attempted) { return attempted >= MIN_DATASETS_FOR_FAILURE - && failed > attempted * PUBLICATION_SEARCH_FAILURE_THRESHOLD; + && failed >= attempted * PUBLICATION_SEARCH_FAILURE_THRESHOLD; } private boolean _test; @@ -373,8 +372,8 @@ private boolean ncbiApiKeyAccepted() /** * @return error when the job could not start, a dataset that should have been sent a reminder was - * not, or the publication search failed for most of the datasets it ran for. Cancelled when the - * job was interrupted with nothing else to report, and complete otherwise. + * not, or the publication search failed for half or more of the datasets it ran for. Cancelled when + * the job was interrupted with nothing else to report, and complete otherwise. */ private TaskStatus postMessage(List expAnnotationIds, Journal panoramaPublic) { @@ -397,7 +396,7 @@ private TaskStatus postMessage(List expAnnotationIds, Journal panoramaP boolean completed = processExperiments(expAnnotationIds, context, processingResults, log); // An ERROR logged through the job's logger sets the status to error, and the status set here - // would overwrite it. Report the errors the run recorded instead. + // would overwrite it. Report the errors the job recorded instead. if (processingResults.getTotalErrors() > 0 || processingResults.publicationSearchFailingWidely()) { return TaskStatus.error; @@ -957,13 +956,13 @@ public void testPublicationSearchFailingWidely() // Below the floor the rate says too little to act on, even when every search failed. assertFalse("2 of 2 is under the floor", publicationSearchFailingWidely(2, 2)); - // At the floor, more than half the searches that ran. + // At the floor, half or more of the searches that ran. assertTrue("2 of 3 is over half", publicationSearchFailingWidely(2, 3)); - assertFalse("1 of 3 is not over half", publicationSearchFailingWidely(1, 3)); + assertFalse("1 of 3 is under half", publicationSearchFailingWidely(1, 3)); - // Exactly half is not enough. The comparison is strict. - assertFalse("2 of 4 is exactly half", publicationSearchFailingWidely(2, 4)); - assertTrue("3 of 4 is over half", publicationSearchFailingWidely(3, 4)); + // Exactly half is enough. + assertTrue("2 of 4 is exactly half", publicationSearchFailingWidely(2, 4)); + assertFalse("1 of 4 is under half", publicationSearchFailingWidely(1, 4)); assertFalse("No failures", publicationSearchFailingWidely(0, 10)); assertFalse("No searches ran", publicationSearchFailingWidely(0, 0)); diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java index f051612f..0d1e216a 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java @@ -112,7 +112,7 @@ private void verifyValidateButton() "NCBI validation result was not displayed", WAIT_FOR_PAGE); String message = result.findElement(getDriver()).getText(); - assertFalse("A key NCBI does not recognise must not be reported as accepted. Message: " + message, + assertFalse("A key NCBI does not recognize must not be reported as accepted. Message: " + message, message.contains("accepted")); } @@ -182,7 +182,6 @@ public void removeTestApiKey() _originalReminderSettings.get("delayUntilFirstReminder"), _originalReminderSettings.get("reminderFrequency"), Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch")), - // A key saved before the run cannot be read back, so leave the key field alone. null); } } diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index b6f1150a..611be61f 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -662,7 +662,9 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de } /** - * @param ncbiApiKey value to enter in the NCBI API key field. Pass null to leave the field untouched. + * Saves the site wide Private Data Reminder Settings. A null ncbiApiKey leaves the key field blank, + * which keeps the key already saved on the server. The page never displays a saved key, so a test + * restoring settings it captured earlier cannot restore the key and should pass null. */ protected void savePrivateDataReminderSettings(String extensionLength, String delayUntilFirstReminder, String reminderFrequency, boolean enablePublicationSearch, String ncbiApiKey) { diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java index 20670d5e..4aa7a48e 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java @@ -488,8 +488,6 @@ public void resetAfterTest() _originalReminderSettings.get("extensionLength"), _originalReminderSettings.get("delayUntilFirstReminder"), _originalReminderSettings.get("reminderFrequency"), - // A key saved before the run cannot be read back to restore it, so leave the - // key field alone here. Any key the test saved is removed below. Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch")), null); } From 6981aebe4c8565f9c7f330c74267a9ac2d56aec1 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 27 Sep 2026 09:53:03 -0700 Subject: [PATCH 16/24] Stopped the reminder settings from requiring an encryption key * PrivateDataReminderSettings.getNcbiApiKeyValue returns no key when the server has no encryption key configured. Reading the encrypted store threw, which failed every call to PrivateDataReminderSettings.get, including extension requests and scheduling the job at startup * saveNcbiApiKey still throws, so an admin saving a key sees the configuration error Co-Authored-By: Claude --- .../panoramapublic/message/PrivateDataReminderSettings.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java index 90c360fc..1ba8b2af 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java +++ b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java @@ -21,6 +21,7 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.data.PropertyManager; +import org.labkey.api.security.Encryption; import org.labkey.api.util.DateUtil; import org.labkey.panoramapublic.model.DatasetStatus; @@ -183,6 +184,11 @@ public static boolean hasNcbiApiKey() private static @Nullable String getNcbiApiKeyValue() { + // The encrypted store throws when no encryption key is configured. Return null in this case. + if (!Encryption.isEncryptionPassPhraseSpecified()) + { + return null; + } Map credentials = PropertyManager.getEncryptedStore().getProperties(PROP_NCBI_CREDENTIALS); return credentials.get(PROP_NCBI_API_KEY); From 37a97f08383ba87433c3e4d0d745113bdd5dc95b Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 27 Sep 2026 10:13:28 -0700 Subject: [PATCH 17/24] Let Cancel in the pipeline UI stop the private data reminder job * PrivateDataReminderJob overrides canInterrupt, so Cancel sets the flag checkInterrupted returns. Before, the loop checked only the thread interrupt flag, which Cancel never sets * processExperiments checks checkInterrupted before each dataset and stops, reporting cancelled Co-Authored-By: Claude --- .../pipeline/PrivateDataReminderJob.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index 86509887..121f8951 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -373,7 +373,7 @@ private boolean ncbiApiKeyAccepted() /** * @return error when the job could not start, a dataset that should have been sent a reminder was * not, or the publication search failed for half or more of the datasets it ran for. Cancelled when - * the job was interrupted with nothing else to report, and complete otherwise. + * the job was cancelled with nothing else to report, and complete otherwise. */ private TaskStatus postMessage(List expAnnotationIds, Journal panoramaPublic) { @@ -405,7 +405,7 @@ private TaskStatus postMessage(List expAnnotationIds, Journal panoramaP } /** - * @return false if the job was interrupted before every dataset was processed. + * @return false if the job was cancelled before every dataset was processed. */ private boolean processExperiments(List expAnnotationIds, ProcessingContext context, ProcessingResults processingResults, Logger log) { @@ -419,11 +419,10 @@ private boolean processExperiments(List expAnnotationIds, ProcessingCon boolean completed = true; for (Integer experimentAnnotationsId : exptIds) { - if (Thread.currentThread().isInterrupted()) + if (checkInterrupted() // checkInterrupted is set by Cancel in the pipeline UI. + || Thread.currentThread().isInterrupted()) { - // An interrupted thread cannot wait, so the NCBI requests would no longer be spaced. - // The remaining datasets would run back to back. - log.warn("Job was interrupted. Stopping before experiment {}.", experimentAnnotationsId); + log.warn("Job was cancelled. Stopping before experiment {}.", experimentAnnotationsId); completed = false; break; } @@ -585,6 +584,12 @@ public String getDescription() return "Post private data reminder messages"; } + @Override + protected boolean canInterrupt() + { + return true; + } + private static class ProcessingContext { private final PrivateDataReminderSettings _reminderSettings; From 34915b86deaaca5bab4322e0b1d2c9d5e8c134be Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 27 Sep 2026 10:54:04 -0700 Subject: [PATCH 18/24] Addressed review findings on NCBI error handling and the reminder settings page * NcbiPublicationSearchServiceImpl.isRetryable retries NoHttpResponseException (connection closed) * errorDetail removes the API key from a 4xx response body before shortening it for the log. Shortening first could cut the key in two and leave its first characters in the log * PrivateDataReminderJob holds the per-dataset transaction only around the post and the DatasetStatus update, not while the NCBI requests run * PrivateDataReminderJob checks the API key after the empty-list return, so a run with no datasets sends no request to NCBI * PrivateDataReminderJob.TestCase turns its logger off, so its ERROR lines stay out of the server log * The NCBI API key field uses autocomplete="new-password", so a browser does not fill in a saved password * "Remove the saved key" is displayed only when a key is saved * savePrivateDataReminderSettings asserts a saved key only for a non-blank key argument Co-Authored-By: Claude --- .../NcbiPublicationSearchServiceImpl.java | 28 ++++++++++++----- .../pipeline/PrivateDataReminderJob.java | 31 ++++++++++++------- .../view/privateDataRemindersSettingsForm.jsp | 4 ++- .../tests/panoramapublic/NcbiApiKeyTest.java | 2 ++ .../PanoramaPublicBaseTest.java | 3 +- 5 files changed, 48 insertions(+), 20 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index 27c46550..14a058a9 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -25,6 +25,7 @@ import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager; import org.apache.hc.client5.http.HttpResponseException; import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.NoHttpResponseException; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.util.Timeout; import org.apache.logging.log4j.Logger; @@ -109,6 +110,9 @@ public static void setInstance(NcbiPublicationSearchService impl) private static final int BAD_REQUEST = 400; // NCBI's response when the API key is not recognized private static final int TOO_MANY_REQUESTS = 429; // NCBI's response when the request rate is exceeded private static final int MAX_ERROR_BODY_CHARS = 500; + // Read past the logged length by more than the length of an NCBI API key (36 characters), so a key + // that crosses the cut is read whole and can be redacted. + private static final int MAX_ERROR_BODY_READ_CHARS = MAX_ERROR_BODY_CHARS + 100; private static final String REDACTED = "REDACTED"; private static final Pattern API_KEY_PARAM = Pattern.compile("api_key=([^&\\s]*)"); @@ -504,7 +508,7 @@ protected String executeGet(String url) throws IOException } /** - * Read the body of a client error response, up to {@link #MAX_ERROR_BODY_CHARS} characters. + * Read the body of a client error response, up to {@link #MAX_ERROR_BODY_READ_CHARS} characters. * 5xx bodies are large, uninformative HTML error pages, so they are skipped. Returns null if * the body cannot be read. */ @@ -517,7 +521,7 @@ protected String executeGet(String url) throws IOException try { - return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8, MAX_ERROR_BODY_CHARS); + return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8, MAX_ERROR_BODY_READ_CHARS); } catch (IOException | org.apache.hc.core5.http.ParseException e) { @@ -534,7 +538,7 @@ static String errorDetail(int status, String reasonPhrase, @Nullable String body { if (status >= 400 && status < 500 && !StringUtils.isBlank(body)) { - return reasonPhrase + " - " + redactApiKey(StringUtils.abbreviate(body.strip(), MAX_ERROR_BODY_CHARS), apiKey); + return reasonPhrase + " - " + StringUtils.abbreviate(redactApiKey(body.strip(), apiKey), MAX_ERROR_BODY_CHARS); } return reasonPhrase; } @@ -566,12 +570,12 @@ static String errorDetail(int status, String reasonPhrase, @Nullable String body } /** - * Read timeouts, connection resets, 5xx responses and 429 are transient NCBI failures worth - * retrying. Other 4xx errors are permanent. + * Read timeouts, connection resets, closed connections, 5xx responses and 429 are transient NCBI + * failures worth retrying. Other 4xx errors are permanent. */ private static boolean isRetryable(IOException e) { - if (e instanceof SocketTimeoutException || e instanceof SocketException) + if (e instanceof SocketTimeoutException || e instanceof SocketException || e instanceof NoHttpResponseException) { return true; } @@ -1676,13 +1680,18 @@ public void testIsRetryable() assertTrue(isRetryable(new HttpResponseException(500, "Internal Server Error"))); assertTrue(isRetryable(new HttpResponseException(503, "Service Unavailable"))); + // A connection reset or closed by the server is transient -> retry + assertTrue("A connection reset should be retried", isRetryable(new SocketException("Connection reset"))); + assertTrue("A connection closed without a response should be retried", + isRetryable(new NoHttpResponseException("The target server failed to respond"))); + // 429 is NCBI's response when the request rate is exceeded, which backoff is for assertTrue(isRetryable(new HttpResponseException(429, "Too Many Requests"))); // Other 4xx and generic IO errors are permanent -> fail fast assertFalse(isRetryable(new HttpResponseException(400, "Bad Request"))); assertFalse(isRetryable(new HttpResponseException(404, "Not Found"))); - assertFalse(isRetryable(new IOException("connection reset"))); + assertFalse(isRetryable(new IOException("Stream closed"))); } @Test @@ -1715,6 +1724,11 @@ public void testErrorDetail() // A body that quotes the request back must not carry the key into the log String echoed = errorDetail(400, "Bad Request", "invalid key SECRET123 for api_key=SECRET123", "SECRET123"); assertFalse("errorDetail must not put the API key in the message", echoed.contains("SECRET123")); + + // A key that spans the truncation point must not leave its prefix in the message + String padding = "x".repeat(MAX_ERROR_BODY_CHARS - 10); + String spanning = errorDetail(400, "Bad Request", padding + " SECRET123456789 trailing", "SECRET123456789"); + assertFalse("errorDetail must not put part of the API key in the message", spanning.contains("SECRET")); } @Test diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index 121f8951..397b82ac 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -16,7 +16,9 @@ package org.labkey.panoramapublic.pipeline; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.config.Configurator; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.Assert; @@ -326,12 +328,6 @@ public void run() return; } - if (!ncbiApiKeyAccepted()) - { - setStatus(TaskStatus.error); - return; - } - setStatus(postMessage(_experimentAnnotationsIds, _panoramaPublic)); } @@ -383,6 +379,10 @@ private TaskStatus postMessage(List expAnnotationIds, Journal panoramaP getLogger().info("No private datasets were found."); return TaskStatus.complete; } + if (!ncbiApiKeyAccepted()) + { + return TaskStatus.error; + } Logger log = getLogger(); ProcessingContext context = ProcessingContext.create(panoramaPublic, getUser(), _test); @@ -427,10 +427,9 @@ private boolean processExperiments(List expAnnotationIds, ProcessingCon break; } - try (DbScope.Transaction transaction = PanoramaPublicManager.getSchema().getScope().ensureTransaction()) + try { processExperiment(experimentAnnotationsId, context, processingResults); - transaction.commit(); } catch (Exception e) { @@ -497,9 +496,13 @@ private void processExperiment(Integer experimentAnnotationsId, ProcessingContex if (!context.isTestMode()) { - postReminderMessage(expAnnotations, submission, announcement, submitter, publicationResult, context); - - updateDatasetStatus(expAnnotations, publicationResult); + // The NCBI requests above run outside the transaction, so it is not held open while they wait. + try (DbScope.Transaction transaction = PanoramaPublicManager.getSchema().getScope().ensureTransaction()) + { + postReminderMessage(expAnnotations, submission, announcement, submitter, publicationResult, context); + updateDatasetStatus(expAnnotations, publicationResult); + transaction.commit(); + } } processingResults.addProcessed(expAnnotations, announcement); @@ -955,6 +958,12 @@ public static class TestCase extends Assert { private static final Logger TEST_LOG = LogHelper.getLogger(TestCase.class, "Private data reminder job tests"); + static + { + // The tests record failures on purpose. Keep their ERROR lines out of the server log. + Configurator.setLevel(TEST_LOG.getName(), Level.OFF); + } + @Test public void testPublicationSearchFailingWidely() { diff --git a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp index 1bdd59f7..b5353d63 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp +++ b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp @@ -226,7 +226,7 @@ <%=h(PrivateDataReminderSettings.PROP_NCBI_API_KEY)%> - " /> <%=button("Validate").onClick("validateNcbiApiKey(); return false;")%> @@ -236,8 +236,10 @@
Create one under Account settings at ncbi.nlm.nih.gov. The saved key is not displayed. Leaving this blank keeps the key that is already saved. + <% if (form.isNcbiApiKeySet()) { %>
+ <% } %> diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java index 0d1e216a..c471ec9e 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java @@ -85,6 +85,8 @@ public void testNcbiApiKeySettings() _savedTestApiKey = false; assertEquals("Remove the saved key should remove it", "false", getPrivateDataReminderSettings().get("ncbiApiKeySaved")); + assertElementNotPresent("Remove the saved key should be offered only when a key is saved", + Locator.checkboxByName("clearNcbiApiKey")); } /** diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index 611be61f..44a39e68 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -16,6 +16,7 @@ package org.labkey.test.tests.panoramapublic; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -696,7 +697,7 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de assertEquals(String.valueOf(extensionLength), getFormElement(Locator.input("extensionLength"))); assertEquals("The saved publication search setting should be displayed on the form", enablePublicationSearch, Locator.checkboxByName("enablePublicationSearch").findElement(getDriver()).isSelected()); - if (ncbiApiKey != null) + if (!StringUtils.isBlank(ncbiApiKey)) { assertEquals("The form should report that a key is saved", "true", getPrivateDataReminderSettings().get("ncbiApiKeySaved")); From 303e4468fabd89794bcf9684b6bc24dbcba6b85a Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 27 Sep 2026 11:28:32 -0700 Subject: [PATCH 19/24] Rejected NCBI API key changes on a server with no encryption key * PrivateDataReminderSettingsAction.validateCommand rejects a new key or a key removal when no encryption key is configured. Before, the other reminder settings were saved, then saving the key threw, and the reminder schedule was not reapplied * The settings page displays a note in place of the key field, the Validate button and the remove checkbox, so Validate no longer reports a key as accepted that cannot be saved Co-Authored-By: Claude --- .../labkey/panoramapublic/PanoramaPublicController.java | 7 +++++++ .../message/PrivateDataReminderSettings.java | 2 ++ .../view/privateDataRemindersSettingsForm.jsp | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java index 346b59d8..d5217f2c 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java @@ -95,6 +95,7 @@ import org.labkey.api.query.QueryView; import org.labkey.api.query.ValidationException; import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.Encryption; import org.labkey.api.security.Group; import org.labkey.api.security.LoginManager; import org.labkey.api.security.MutableSecurityPolicy; @@ -10182,6 +10183,12 @@ else if (PrivateDataReminderSettings.parseReminderTime(form.getReminderTime()) = { errors.reject(ERROR_MSG, "Enter a new NCBI API key or select 'Remove the saved key', not both."); } + // saveNcbiApiKey throws without an encryption key. Reject here, before handlePost saves the other settings. + if ((form.isClearNcbiApiKey() || !StringUtils.isBlank(form.getNcbiApiKey())) + && !Encryption.isEncryptionPassPhraseSpecified()) + { + errors.reject(ERROR_MSG, PrivateDataReminderSettings.NCBI_API_KEY_REQUIRES_ENCRYPTION); + } } @Override diff --git a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java index 1ba8b2af..c04892f2 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java +++ b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java @@ -46,6 +46,8 @@ public class PrivateDataReminderSettings public static final String PROP_PUBLICATION_SEARCH_FREQUENCY = "Publication search frequency (months)"; public static final String PROP_NCBI_API_KEY = "NCBI API key"; public static final String PROP_NCBI_CREDENTIALS = "Panorama Public NCBI credentials"; + public static final String NCBI_API_KEY_REQUIRES_ENCRYPTION = "An NCBI API key cannot be saved because this server" + + " has no encryption key configured."; private static final boolean DEFAULT_ENABLE_REMINDERS = false; public static final String DEFAULT_REMINDER_TIME = "8:00 AM"; diff --git a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp index b5353d63..7249d894 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp +++ b/panoramapublic/src/org/labkey/panoramapublic/view/privateDataRemindersSettingsForm.jsp @@ -16,6 +16,7 @@ */ %> <%@ taglib prefix="labkey" uri="http://www.labkey.org/taglib" %> +<%@ page import="org.labkey.api.security.Encryption" %> <%@ page import="org.labkey.api.view.HttpView" %> <%@ page import="org.labkey.api.view.JspView" %> <%@ page import="org.labkey.api.view.template.ClientDependencies" %> @@ -226,6 +227,9 @@ <%=h(PrivateDataReminderSettings.PROP_NCBI_API_KEY)%> + <% if (!Encryption.isEncryptionPassPhraseSpecified()) { %> +
<%=h(PrivateDataReminderSettings.NCBI_API_KEY_REQUIRES_ENCRYPTION)%>
+ <% } else { %> " /> @@ -241,6 +245,7 @@ <% } %> + <% } %> From 45b4e79b0852bb47cc4938171f5ed72535939128 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 27 Sep 2026 14:39:50 -0700 Subject: [PATCH 20/24] Moved the NCBI HTTP request code into NcbiHttpClient * NcbiHttpClient holds the request, retry, error message and API key redaction code and constants that were in NcbiPublicationSearchServiceImpl, with their unit tests. No behavior change * NcbiPublicationSearchServiceImpl sends its requests through an NcbiHttpClient * MockNcbiPublicationSearchService supplies an NcbiHttpClient that returns canned responses from executeGet, so the request loop in NcbiHttpClient.getString runs in the Selenium tests on TeamCity * Added a unit test that checks the mock returns registered data through NcbiHttpClient Co-Authored-By: Claude --- .../panoramapublic/PanoramaPublicModule.java | 2 + .../MockNcbiPublicationSearchService.java | 207 ++++---- .../panoramapublic/ncbi/NcbiHttpClient.java | 458 ++++++++++++++++++ .../NcbiPublicationSearchServiceImpl.java | 448 ++--------------- 4 files changed, 606 insertions(+), 509 deletions(-) create mode 100644 panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiHttpClient.java diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java index 791d165a..b5b1f3d6 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java @@ -50,6 +50,7 @@ import org.labkey.panoramapublic.catalog.CatalogImageAttachmentType; import org.labkey.panoramapublic.message.PrivateDataMessageScheduler; import org.labkey.panoramapublic.message.PrivateDataReminderSettings; +import org.labkey.panoramapublic.ncbi.NcbiHttpClient; import org.labkey.panoramapublic.ncbi.NcbiPublicationSearchServiceImpl; import org.labkey.panoramapublic.model.Journal; import org.labkey.panoramapublic.model.speclib.SpecLibKey; @@ -399,6 +400,7 @@ public Set getSchemaNames() set.add(PrivateDataReminderSettings.TestCase.class); set.add(PrivateDataReminderJob.TestCase.class); set.add(NcbiPublicationSearchServiceImpl.TestCase.class); + set.add(NcbiHttpClient.TestCase.class); set.add(NcbiUtils.TestCase.class); return set; diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java index af26e980..9de89510 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java @@ -15,7 +15,6 @@ */ package org.labkey.panoramapublic.ncbi; -import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.json.JSONArray; import org.json.JSONObject; @@ -32,30 +31,32 @@ /** * Mock implementation of {@link NcbiPublicationSearchService} that returns canned data registered by tests. * Used by Selenium tests when running on TeamCity. - * Extends {@link NcbiPublicationSearchServiceImpl} and only overrides {@link #getString(String, Logger)}, - * the method every NCBI request passes through. The override takes the place of the real HTTP request - * in {@code executeGet()} and the retry loop around it. All search logic, filtering, author/title - * verification, citation parsing, and priority filtering run through the real implementation code. + * Extends {@link NcbiPublicationSearchServiceImpl} and gives it an {@link NcbiHttpClient} whose + * {@code executeGet()} returns canned responses in place of the real HTTP request. The retry loop in + * {@link NcbiHttpClient#getString}, and all search logic, filtering, author/title verification, citation + * parsing, and priority filtering run through the real implementation code. * Tests register mock articles via {@link #register}, providing the database, ID, search key, * metadata fields, and citation. The mock builds internal lookup maps from this data and returns - * appropriate responses when the real search logic calls {@code getString()}. + * appropriate responses when the real search logic sends a request. */ public class MockNcbiPublicationSearchService extends NcbiPublicationSearchServiceImpl { - // ESearch: searchKey -> list of IDs (per database) - private final Map> _pmcSearchResults = new HashMap<>(); - private final Map> _pubmedSearchResults = new HashMap<>(); + private final CannedResponses _responses; - // ESummary: ID -> metadata JSONObject (per database) - private final Map _pmcMetadata = new HashMap<>(); - private final Map _pubmedMetadata = new HashMap<>(); + public MockNcbiPublicationSearchService() + { + this(new CannedResponses()); + } - // Citations: PMID -> citation string - private final Map _citations = new HashMap<>(); + private MockNcbiPublicationSearchService(CannedResponses responses) + { + super(responses); + _responses = responses; + } /** * Register a mock article. The mock stores the data in internal lookup maps used by - * {@link #getString(String, Logger)}. + * {@link CannedResponses#executeGet(String)}. * @param database "pmc" or "pubmed" — the NCBI database this article is in * @param id the article ID in the given database (numeric ID for pmc or pubmed) * @param searchKey what ESearch query term finds this article (e.g. PXD ID for PMC, author last name for PubMed) @@ -110,13 +111,13 @@ public void register(String database, String id, String searchKey, // Store in appropriate maps if (isPmc) { - _pmcSearchResults.computeIfAbsent(searchKey, k -> new ArrayList<>()).add(id); - _pmcMetadata.put(id, metadata); + _responses._pmcSearchResults.computeIfAbsent(searchKey, k -> new ArrayList<>()).add(id); + _responses._pmcMetadata.put(id, metadata); } else { - _pubmedSearchResults.computeIfAbsent(searchKey, k -> new ArrayList<>()).add(id); - _pubmedMetadata.put(id, metadata); + _responses._pubmedSearchResults.computeIfAbsent(searchKey, k -> new ArrayList<>()).add(id); + _responses._pubmedMetadata.put(id, metadata); } // Store citation keyed by PMID. @@ -127,113 +128,127 @@ public void register(String database, String id, String searchKey, String citationKey = isPmc ? pmid : id; if (citationKey != null) { - _citations.put(citationKey, citation); + _responses._citations.put(citationKey, citation); } } } /** - * Returns canned responses for NCBI API requests based on registered mock data. - * Handles ESearch, ESummary, and Citation Exporter URLs. + * An {@link NcbiHttpClient} that returns canned responses for NCBI API requests based on registered + * mock data. Handles ESearch, ESummary, and Citation Exporter URLs. */ - @Override - protected String getString(String url, Logger log) throws IOException + private static class CannedResponses extends NcbiHttpClient { - if (url.contains("esearch.fcgi")) - { - return handleESearch(url).toString(); - } - else if (url.contains("esummary.fcgi")) - { - return handleESummary(url).toString(); - } - else if (url.contains("lit/ctxp")) + // ESearch: searchKey -> list of IDs (per database) + private final Map> _pmcSearchResults = new HashMap<>(); + private final Map> _pubmedSearchResults = new HashMap<>(); + + // ESummary: ID -> metadata JSONObject (per database) + private final Map _pmcMetadata = new HashMap<>(); + private final Map _pubmedMetadata = new HashMap<>(); + + // Citations: PMID -> citation string + private final Map _citations = new HashMap<>(); + + @Override + protected String executeGet(String url) throws IOException { - return handleCitation(url).toString(); + if (url.contains("esearch.fcgi")) + { + return handleESearch(url).toString(); + } + else if (url.contains("esummary.fcgi")) + { + return handleESummary(url).toString(); + } + else if (url.contains("lit/ctxp")) + { + return handleCitation(url).toString(); + } + throw new IOException("MockNcbiPublicationSearchService: unexpected URL: " + url); } - throw new IOException("MockNcbiPublicationSearchService: unexpected URL: " + url); - } - private JSONObject handleESearch(String url) - { - boolean isPmc = "pmc".equals(extractQueryParam(url, "db")); - Map> searchMap = isPmc ? _pmcSearchResults : _pubmedSearchResults; + private JSONObject handleESearch(String url) + { + boolean isPmc = "pmc".equals(extractQueryParam(url, "db")); + Map> searchMap = isPmc ? _pmcSearchResults : _pubmedSearchResults; - // Match search keys against the decoded ESearch query term, so a key cannot match part of - // another parameter such as tool or email. - String term = extractQueryParam(url, "term"); + // Match search keys against the decoded ESearch query term, so a key cannot match part of + // another parameter such as tool or email. + String term = extractQueryParam(url, "term"); - JSONArray idList = new JSONArray(); - if (term != null) - { - for (Map.Entry> entry : searchMap.entrySet()) + JSONArray idList = new JSONArray(); + if (term != null) { - // A key is a substring of the term. searchPmc wraps the PMC term in quotes, e.g. "PXD056793". - if (term.contains(entry.getKey())) + for (Map.Entry> entry : searchMap.entrySet()) { - entry.getValue().forEach(idList::put); + // A key is a substring of the term. searchPmc wraps the PMC term in quotes, e.g. "PXD056793". + if (term.contains(entry.getKey())) + { + entry.getValue().forEach(idList::put); + } } } - } - JSONObject esearchResult = new JSONObject(); - esearchResult.put("idlist", idList); - return new JSONObject().put("esearchresult", esearchResult); - } + JSONObject esearchResult = new JSONObject(); + esearchResult.put("idlist", idList); + return new JSONObject().put("esearchresult", esearchResult); + } - private JSONObject handleESummary(String url) - { - boolean isPmc = "pmc".equals(extractQueryParam(url, "db")); - Map metadataMap = isPmc ? _pmcMetadata : _pubmedMetadata; + private JSONObject handleESummary(String url) + { + boolean isPmc = "pmc".equals(extractQueryParam(url, "db")); + Map metadataMap = isPmc ? _pmcMetadata : _pubmedMetadata; - // ESummary requests a comma-separated list of IDs in the "id" parameter. Match registered - // IDs against that list rather than scanning the whole URL. - String idParam = extractQueryParam(url, "id"); - List requestedIds = idParam == null ? List.of() : Arrays.asList(idParam.split(",")); + // ESummary requests a comma-separated list of IDs in the "id" parameter. Match registered + // IDs against that list rather than scanning the whole URL. + String idParam = extractQueryParam(url, "id"); + List requestedIds = idParam == null ? List.of() : Arrays.asList(idParam.split(",")); - JSONObject result = new JSONObject(); - for (Map.Entry entry : metadataMap.entrySet()) - { - if (requestedIds.contains(entry.getKey())) + JSONObject result = new JSONObject(); + for (Map.Entry entry : metadataMap.entrySet()) { - result.put(entry.getKey(), entry.getValue()); + if (requestedIds.contains(entry.getKey())) + { + result.put(entry.getKey(), entry.getValue()); + } } - } - return new JSONObject().put("result", result); - } + return new JSONObject().put("result", result); + } - /** - * Build a citation JSON response matching the NCBI Literature Citation Exporter format. - * The real API returns {@code {"nlm":{"orig":"citation text..."}}}. - * If no citation is registered for the ID, returns an empty JSON object. - */ - private JSONObject handleCitation(String url) - { - String id = extractQueryParam(url, "id"); - String citation = id != null ? _citations.get(id) : null; - if (citation != null) + /** + * Build a citation JSON response matching the NCBI Literature Citation Exporter format. + * The real API returns {@code {"nlm":{"orig":"citation text..."}}}. + * If no citation is registered for the ID, returns an empty JSON object. + */ + private JSONObject handleCitation(String url) { - return new JSONObject().put("nlm", new JSONObject().put("orig", citation)); + String id = extractQueryParam(url, "id"); + String citation = id != null ? _citations.get(id) : null; + if (citation != null) + { + return new JSONObject().put("nlm", new JSONObject().put("orig", citation)); + } + return new JSONObject(); } - return new JSONObject(); - } - /** - * Returns the URL-decoded value of the given query parameter, or null if it is not present. - */ - private static @Nullable String extractQueryParam(String url, String name) - { - int queryStart = url.indexOf('?'); - String query = queryStart >= 0 ? url.substring(queryStart + 1) : url; - for (String pair : query.split("&")) + /** + * Returns the URL-decoded value of the given query parameter, or null if it is not present. + */ + private static @Nullable String extractQueryParam(String url, String name) { - int eq = pair.indexOf('='); - if (eq > 0 && pair.substring(0, eq).equals(name)) + int queryStart = url.indexOf('?'); + String query = queryStart >= 0 ? url.substring(queryStart + 1) : url; + for (String pair : query.split("&")) { - return URLDecoder.decode(pair.substring(eq + 1), StandardCharsets.UTF_8); + int eq = pair.indexOf('='); + if (eq > 0 && pair.substring(0, eq).equals(name)) + { + return URLDecoder.decode(pair.substring(eq + 1), StandardCharsets.UTF_8); + } } + return null; } - return null; } } diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiHttpClient.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiHttpClient.java new file mode 100644 index 00000000..49ea1d5c --- /dev/null +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiHttpClient.java @@ -0,0 +1,458 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.panoramapublic.ncbi; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hc.client5.http.HttpResponseException; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager; +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.NoHttpResponseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.util.Timeout; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.util.logging.LogHelper; + +import java.io.IOException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Sends HTTP GET requests to NCBI for {@link NcbiPublicationSearchServiceImpl}. Retries transient + * failures, and removes the NCBI API key from anything it logs or puts in an exception message. + */ +public class NcbiHttpClient +{ + private static final Logger LOG = LogHelper.getLogger(NcbiHttpClient.class, "HTTP requests to NCBI for the Panorama Public publication search"); + + private static final int TIMEOUT_MS = 10000; // 10 seconds + + // NCBI eutils fails intermittently, even well under the rate limit. Retry the transient + // failures listed on isRetryable a few times with exponential backoff before giving up. + private static final int MAX_HTTP_ATTEMPTS = 3; // initial try + 2 retries + private static final int RETRY_BASE_DELAY_MS = 500; // exponential backoff base + + private static final int TOO_MANY_REQUESTS = 429; // NCBI's response when the request rate is exceeded + private static final int MAX_ERROR_BODY_CHARS = 500; + // Read past the logged length by more than the length of an NCBI API key (36 characters), so a key + // that crosses the cut is read whole and can be redacted. + private static final int MAX_ERROR_BODY_READ_CHARS = MAX_ERROR_BODY_CHARS + 100; + + private static final String REDACTED = "REDACTED"; + private static final Pattern API_KEY_PARAM = Pattern.compile("api_key=([^&\\s]*)"); + + /** + * Execute an HTTP GET request and return the response body as a string. Retry warnings are + * written to {@code log}. + * @throws IOException if the request fails or the server returns a non-2xx response + */ + public String getString(String url, @NotNull Logger log) throws IOException + { + for (int attempt = 1; ; attempt++) + { + try + { + return executeGet(url); + } + catch (IOException e) + { + if (attempt >= MAX_HTTP_ATTEMPTS || !isRetryable(e)) + { + throw e; + } + long delayMs = retryDelayMs(attempt); + log.warn("NCBI request failed (attempt {} of {}). Retrying in {} ms. URL: {}. Cause: {}", + attempt, MAX_HTTP_ATTEMPTS, delayMs, redactApiKey(url, apiKeyFrom(url)), e.toString()); + if (!sleepMs(delayMs)) + { + // An interrupted thread cannot wait, so the remaining attempts would run back to + // back. Give up instead. + throw e; + } + } + } + } + + // Overridden in unit tests to drive the retry loop in getString(), and by + // MockNcbiPublicationSearchService to return canned responses, without real HTTP calls. + protected String executeGet(String url) throws IOException + { + ConnectionConfig connectionConfig = ConnectionConfig.custom() + .setConnectTimeout(Timeout.ofMilliseconds(TIMEOUT_MS)) + .setSocketTimeout(Timeout.ofMilliseconds(TIMEOUT_MS)) + .build(); + + RequestConfig requestConfig = RequestConfig.custom() + .setResponseTimeout(Timeout.ofMilliseconds(TIMEOUT_MS)) + .build(); + + BasicHttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(); + connectionManager.setConnectionConfig(connectionConfig); + + try (CloseableHttpClient client = HttpClientBuilder.create() + .setDefaultRequestConfig(requestConfig) + .setConnectionManager(connectionManager) + // Without this, the number of requests would be up to twice MAX_HTTP_ATTEMPTS. isRetryable + // has to cover what this turns off. + .disableAutomaticRetries() + .build()) + { + HttpGet getRequest = new HttpGet(url); + return client.execute(getRequest, response -> { + int status = response.getCode(); + if (status < 200 || status >= 300) + { + throw new HttpResponseException(status, + errorDetail(status, response.getReasonPhrase(), readErrorBody(status, response), + apiKeyFrom(url))); + } + return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + }); + } + } + + /** + * Read the body of a client error response, up to {@link #MAX_ERROR_BODY_READ_CHARS} characters. + * 5xx bodies are large, uninformative HTML error pages, so they are skipped. Returns null if + * the body cannot be read. + */ + private static @Nullable String readErrorBody(int status, ClassicHttpResponse response) + { + if (status < 400 || status >= 500 || response.getEntity() == null) + { + return null; + } + + try + { + return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8, MAX_ERROR_BODY_READ_CHARS); + } + catch (IOException | org.apache.hc.core5.http.ParseException e) + { + return null; + } + } + + /** + * Build the message for a non-2xx HttpResponseException. A 4xx appends the response body, so + * NCBI's reason (e.g. "API key invalid") reaches the log. Other statuses use only the reason + * phrase. The API key is removed from the body, which is third-party text bound for a log. + */ + static String errorDetail(int status, String reasonPhrase, @Nullable String body, @Nullable String apiKey) + { + if (status >= 400 && status < 500 && !StringUtils.isBlank(body)) + { + return reasonPhrase + " - " + StringUtils.abbreviate(redactApiKey(body.strip(), apiKey), MAX_ERROR_BODY_CHARS); + } + return reasonPhrase; + } + + /** + * Replace the NCBI API key wherever it appears in text bound for a log. + */ + static @Nullable String redactApiKey(@Nullable String text, @Nullable String apiKey) + { + if (text == null) + { + return null; + } + String redacted = text.replaceAll("(api_key=)[^&\\s]*", "$1" + REDACTED); + if (!StringUtils.isBlank(apiKey)) + { + redacted = redacted.replace(apiKey.trim(), REDACTED); + } + return redacted; + } + + /** + * Returns the value of the api_key query parameter in the given URL, or null if there is none. + */ + static @Nullable String apiKeyFrom(String url) + { + Matcher matcher = API_KEY_PARAM.matcher(url); + return matcher.find() ? matcher.group(1) : null; + } + + /** + * Read timeouts, connection resets, closed connections, 5xx responses and 429 are transient NCBI + * failures worth retrying. Other 4xx errors are permanent. + */ + private static boolean isRetryable(IOException e) + { + if (e instanceof SocketTimeoutException || e instanceof SocketException || e instanceof NoHttpResponseException) + { + return true; + } + if (e instanceof HttpResponseException hre) + { + return hre.getStatusCode() >= 500 || hre.getStatusCode() == TOO_MANY_REQUESTS; + } + return false; + } + + // 500ms after the first failure, then doubling. + private static long retryDelayMs(int attempt) + { + return (long) RETRY_BASE_DELAY_MS << (attempt - 1); + } + + /** + * @return false if the thread was interrupted, in which case the caller should stop rather than + * carry on without the delay it asked for. + */ + protected boolean sleepMs(long ms) + { + try + { + Thread.sleep(ms); + return true; + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return false; + } + } + + public static class TestCase extends Assert + { + @Test + public void testIsRetryable() + { + // Read timeouts and 5xx responses are transient NCBI failures -> retry + assertTrue(isRetryable(new SocketTimeoutException("Read timed out"))); + assertTrue(isRetryable(new HttpResponseException(500, "Internal Server Error"))); + assertTrue(isRetryable(new HttpResponseException(503, "Service Unavailable"))); + + // A connection reset or closed by the server is transient -> retry + assertTrue("A connection reset should be retried", isRetryable(new SocketException("Connection reset"))); + assertTrue("A connection closed without a response should be retried", + isRetryable(new NoHttpResponseException("The target server failed to respond"))); + + // 429 is NCBI's response when the request rate is exceeded, which backoff is for + assertTrue(isRetryable(new HttpResponseException(429, "Too Many Requests"))); + + // Other 4xx and generic IO errors are permanent -> fail fast + assertFalse(isRetryable(new HttpResponseException(400, "Bad Request"))); + assertFalse(isRetryable(new HttpResponseException(404, "Not Found"))); + assertFalse(isRetryable(new IOException("Stream closed"))); + } + + @Test + public void testRetryDelayMs() + { + // Exponential backoff of 500ms then 1000ms. With MAX_HTTP_ATTEMPTS at 3 the loop sleeps + // after the first two failures and rethrows after the third, so those are the only + // delays a request can wait. + assertEquals(500, retryDelayMs(1)); + assertEquals(1000, retryDelayMs(2)); + assertEquals("A request sleeps once per failed attempt except the last, so only the" + + " delays asserted above are reachable", 2, MAX_HTTP_ATTEMPTS - 1); + } + + @Test + public void testErrorDetail() + { + // 4xx: the response body is appended so the cause (e.g. an invalid API key) is logged + String detail = errorDetail(400, "Bad Request", "{\"error\":\"API key invalid\"}", null); + assertTrue(detail.contains("Bad Request")); + assertTrue(detail.contains("API key invalid")); + + // 5xx: body omitted (uninformative) + assertEquals("Internal Server Error", errorDetail(500, "Internal Server Error", "oops", null)); + + // 4xx with blank or null body: just the reason phrase, no trailing separator + assertEquals("Bad Request", errorDetail(400, "Bad Request", "", null)); + assertEquals("Bad Request", errorDetail(400, "Bad Request", null, null)); + + // A body that quotes the request back must not carry the key into the log + String echoed = errorDetail(400, "Bad Request", "invalid key SECRET123 for api_key=SECRET123", "SECRET123"); + assertFalse("errorDetail must not put the API key in the message", echoed.contains("SECRET123")); + + // A key that spans the truncation point must not leave its prefix in the message + String padding = "x".repeat(MAX_ERROR_BODY_CHARS - 10); + String spanning = errorDetail(400, "Bad Request", padding + " SECRET123456789 trailing", "SECRET123456789"); + assertFalse("errorDetail must not put part of the API key in the message", spanning.contains("SECRET")); + } + + @Test + public void testRedactApiKey() + { + // The key is stripped from an eutils URL, and the rest of the URL is left intact + String url = "https://eutils.ncbi.nlm.nih.gov/esearch.fcgi?db=pmc&api_key=SECRET123&term=PXD001"; + String redacted = redactApiKey(url, "SECRET123"); + assertFalse("The redacted URL must not contain the key", redacted.contains("SECRET123")); + assertTrue("The redacted URL must keep its other parameters", redacted.contains("term=PXD001")); + assertTrue(redacted.contains("db=pmc")); + + // The key is stripped even when it appears without the api_key= prefix + assertFalse(redactApiKey("rejected key SECRET123", "SECRET123").contains("SECRET123")); + + // A URL with no key is unchanged, and null text stays null + String noKey = "https://eutils.ncbi.nlm.nih.gov/esearch.fcgi?db=pmc&term=PXD001"; + assertEquals(noKey, redactApiKey(noKey, null)); + assertNull(redactApiKey(null, "SECRET123")); + + // The key is recovered from the URL so callers do not have to read the settings + assertEquals("SECRET123", apiKeyFrom(url)); + assertNull(apiKeyFrom(noKey)); + } + + @Test + public void testGetStringRetriesTransientFailures() throws IOException + { + // executeGet returns a 5xx twice, then succeeds. getString should retry and return the body. + int[] attempts = {0}; + NoWaitClient client = new NoWaitClient() + { + @Override + protected String executeGet(String url) throws IOException + { + if (++attempts[0] < 3) + throw new HttpResponseException(503, "Service Unavailable"); + return "body"; + } + }; + assertEquals("body", client.getString("http://test", LOG)); + assertEquals("Should retry until the 3rd attempt succeeds", 3, attempts[0]); + assertEquals("The loop should wait 500ms then 1000ms", List.of(500L, 1000L), client.sleeps); + } + + @Test + public void testGetStringStopsWhenInterrupted() + { + // An interrupted thread cannot wait, so retrying would send the remaining attempts back + // to back. getString should give up after the first failure instead. + int[] attempts = {0}; + NcbiHttpClient client = new NcbiHttpClient() + { + @Override + protected String executeGet(String url) throws IOException + { + attempts[0]++; + Thread.currentThread().interrupt(); + throw new HttpResponseException(503, "Service Unavailable"); + } + }; + + try + { + client.getString("http://test", LOG); + fail("Expected the interrupted request to be rethrown"); + } + catch (IOException expected) + { + assertEquals("An interrupted request should not be retried", 1, attempts[0]); + } + finally + { + // Clear the flag so it cannot reach whatever test runs next. + Thread.interrupted(); + } + } + + @Test + public void testGetStringGivesUpAfterMaxAttempts() + { + // executeGet always returns a 5xx. getString should try MAX_HTTP_ATTEMPTS times, then rethrow. + int[] attempts = {0}; + NoWaitClient client = new NoWaitClient() + { + @Override + protected String executeGet(String url) throws IOException + { + attempts[0]++; + throw new HttpResponseException(500, "Internal Server Error"); + } + }; + try + { + client.getString("http://test", LOG); + fail("Expected HttpResponseException after exhausting retries"); + } + catch (HttpResponseException e) + { + assertEquals(500, e.getStatusCode()); + } + catch (IOException e) + { + fail("Expected HttpResponseException, got " + e); + } + assertEquals("Should attempt exactly MAX_HTTP_ATTEMPTS times", MAX_HTTP_ATTEMPTS, attempts[0]); + assertEquals("One wait fewer than attempts, and no wait after the last", List.of(500L, 1000L), client.sleeps); + } + + @Test + public void testGetStringDoesNotRetryClientErrors() + { + // A 4xx is permanent. getString should fail immediately without retrying. + int[] attempts = {0}; + NoWaitClient client = new NoWaitClient() + { + @Override + protected String executeGet(String url) throws IOException + { + attempts[0]++; + throw new HttpResponseException(400, "Bad Request"); + } + }; + try + { + client.getString("http://test", LOG); + fail("Expected HttpResponseException for a 4xx"); + } + catch (HttpResponseException e) + { + assertEquals(400, e.getStatusCode()); + } + catch (IOException e) + { + fail("Expected HttpResponseException, got " + e); + } + assertEquals("4xx must not be retried", 1, attempts[0]); + assertTrue("A 4xx must not wait", client.sleeps.isEmpty()); + } + + /** + * Runs the retry loop without waiting, and records the delays it asked for. A test can then + * check the delays the loop used, which retryDelayMs on its own cannot show. + */ + static class NoWaitClient extends NcbiHttpClient + { + private final List sleeps = new ArrayList<>(); + + @Override + protected boolean sleepMs(long ms) + { + sleeps.add(ms); + return true; + } + } + } +} diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index 14a058a9..fe3d0616 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -17,17 +17,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; -import org.apache.hc.client5.http.classic.methods.HttpGet; -import org.apache.hc.client5.http.config.ConnectionConfig; -import org.apache.hc.client5.http.config.RequestConfig; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager; import org.apache.hc.client5.http.HttpResponseException; -import org.apache.hc.core5.http.ClassicHttpResponse; -import org.apache.hc.core5.http.NoHttpResponseException; -import org.apache.hc.core5.http.io.entity.EntityUtils; -import org.apache.hc.core5.util.Timeout; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -45,7 +35,6 @@ import org.labkey.panoramapublic.ncbi.NcbiConstants.DB; import java.io.IOException; -import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -62,8 +51,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.labkey.panoramapublic.ncbi.PublicationMatch.MATCH_DOI; @@ -90,6 +77,18 @@ public static void setInstance(NcbiPublicationSearchService impl) _instance = impl; } + private final NcbiHttpClient _httpClient; + + public NcbiPublicationSearchServiceImpl() + { + this(new NcbiHttpClient()); + } + + NcbiPublicationSearchServiceImpl(NcbiHttpClient httpClient) + { + _httpClient = httpClient; + } + // NCBI API endpoints private static final String ESEARCH_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"; private static final String ESUMMARY_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"; @@ -100,22 +99,8 @@ public static void setInstance(NcbiPublicationSearchService impl) // API parameters private static final int RATE_LIMIT_DELAY_MS = 400; // NCBI allows 3 requests/sec without an API key - private static final int TIMEOUT_MS = 10000; // 10 seconds - - // NCBI eutils fails intermittently, even well under the rate limit. Retry the transient - // failures listed on isRetryable a few times with exponential backoff before giving up. - private static final int MAX_HTTP_ATTEMPTS = 3; // initial try + 2 retries - private static final int RETRY_BASE_DELAY_MS = 500; // exponential backoff base private static final int BAD_REQUEST = 400; // NCBI's response when the API key is not recognized - private static final int TOO_MANY_REQUESTS = 429; // NCBI's response when the request rate is exceeded - private static final int MAX_ERROR_BODY_CHARS = 500; - // Read past the logged length by more than the length of an NCBI API key (36 characters), so a key - // that crosses the cut is read whole and can be redacted. - private static final int MAX_ERROR_BODY_READ_CHARS = MAX_ERROR_BODY_CHARS + 100; - - private static final String REDACTED = "REDACTED"; - private static final Pattern API_KEY_PARAM = Pattern.compile("api_key=([^&\\s]*)"); // NCBI suggests using the 'tool' and 'email' parameters on E-utilities URLs // https://www.nlm.nih.gov/dataguide/eutilities/utilities.html @@ -417,14 +402,14 @@ private List executeSearch(String query, String database, Logger log) { // NCBI rejects an unrecognized key with a 400. Any other status says nothing about the // key, including a 403 or 404 from a proxy between the server and NCBI. - String message = redactApiKey(e.getMessage(), apiKey); + String message = NcbiHttpClient.redactApiKey(e.getMessage(), apiKey); return e.getStatusCode() == BAD_REQUEST ? NcbiApiKeyCheck.rejected(message) : NcbiApiKeyCheck.unconfirmed(message); } catch (IOException e) { - return NcbiApiKeyCheck.unconfirmed(redactApiKey(e.getMessage(), apiKey)); + return NcbiApiKeyCheck.unconfirmed(NcbiHttpClient.redactApiKey(e.getMessage(), apiKey)); } } @@ -433,181 +418,14 @@ private List executeSearch(String query, String database, Logger log) * @throws IOException if the request fails or the server returns a non-2xx response * @throws JSONException if the response body is not valid JSON */ - protected JSONObject getJson(String url, Logger log) throws IOException + private JSONObject getJson(String url, Logger log) throws IOException { return new JSONObject(getString(url, log)); } - /** - * Execute an HTTP GET request and return the response body as a string. Retry warnings are - * written to {@code log}. - * @throws IOException if the request fails or the server returns a non-2xx response - */ - protected String getString(String url, Logger log) throws IOException + private String getString(String url, @Nullable Logger log) throws IOException { - for (int attempt = 1; ; attempt++) - { - try - { - return executeGet(url); - } - catch (IOException e) - { - if (attempt >= MAX_HTTP_ATTEMPTS || !isRetryable(e)) - { - throw e; - } - long delayMs = retryDelayMs(attempt); - getLog(log).warn("NCBI request failed (attempt {} of {}). Retrying in {} ms. URL: {}. Cause: {}", - attempt, MAX_HTTP_ATTEMPTS, delayMs, redactApiKey(url, apiKeyFrom(url)), e.toString()); - if (!sleepMs(delayMs)) - { - // An interrupted thread cannot wait, so the remaining attempts would run back to - // back. Give up instead. - throw e; - } - } - } - } - - // Overridden in unit tests to drive the retry loop in getString() without real HTTP calls. - protected String executeGet(String url) throws IOException - { - ConnectionConfig connectionConfig = ConnectionConfig.custom() - .setConnectTimeout(Timeout.ofMilliseconds(TIMEOUT_MS)) - .setSocketTimeout(Timeout.ofMilliseconds(TIMEOUT_MS)) - .build(); - - RequestConfig requestConfig = RequestConfig.custom() - .setResponseTimeout(Timeout.ofMilliseconds(TIMEOUT_MS)) - .build(); - - BasicHttpClientConnectionManager connectionManager = new BasicHttpClientConnectionManager(); - connectionManager.setConnectionConfig(connectionConfig); - - try (CloseableHttpClient client = HttpClientBuilder.create() - .setDefaultRequestConfig(requestConfig) - .setConnectionManager(connectionManager) - // Without this, the number of requests would be up to twice MAX_HTTP_ATTEMPTS. isRetryable - // has to cover what this turns off. - .disableAutomaticRetries() - .build()) - { - HttpGet getRequest = new HttpGet(url); - return client.execute(getRequest, response -> { - int status = response.getCode(); - if (status < 200 || status >= 300) - { - throw new HttpResponseException(status, - errorDetail(status, response.getReasonPhrase(), readErrorBody(status, response), - apiKeyFrom(url))); - } - return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); - }); - } - } - - /** - * Read the body of a client error response, up to {@link #MAX_ERROR_BODY_READ_CHARS} characters. - * 5xx bodies are large, uninformative HTML error pages, so they are skipped. Returns null if - * the body cannot be read. - */ - private static @Nullable String readErrorBody(int status, ClassicHttpResponse response) - { - if (status < 400 || status >= 500 || response.getEntity() == null) - { - return null; - } - - try - { - return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8, MAX_ERROR_BODY_READ_CHARS); - } - catch (IOException | org.apache.hc.core5.http.ParseException e) - { - return null; - } - } - - /** - * Build the message for a non-2xx HttpResponseException. A 4xx appends the response body, so - * NCBI's reason (e.g. "API key invalid") reaches the log. Other statuses use only the reason - * phrase. The API key is removed from the body, which is third-party text bound for a log. - */ - static String errorDetail(int status, String reasonPhrase, @Nullable String body, @Nullable String apiKey) - { - if (status >= 400 && status < 500 && !StringUtils.isBlank(body)) - { - return reasonPhrase + " - " + StringUtils.abbreviate(redactApiKey(body.strip(), apiKey), MAX_ERROR_BODY_CHARS); - } - return reasonPhrase; - } - - /** - * Replace the NCBI API key wherever it appears in text bound for a log. - */ - static @Nullable String redactApiKey(@Nullable String text, @Nullable String apiKey) - { - if (text == null) - { - return null; - } - String redacted = text.replaceAll("(api_key=)[^&\\s]*", "$1" + REDACTED); - if (!StringUtils.isBlank(apiKey)) - { - redacted = redacted.replace(apiKey.trim(), REDACTED); - } - return redacted; - } - - /** - * Returns the value of the api_key query parameter in the given URL, or null if there is none. - */ - static @Nullable String apiKeyFrom(String url) - { - Matcher matcher = API_KEY_PARAM.matcher(url); - return matcher.find() ? matcher.group(1) : null; - } - - /** - * Read timeouts, connection resets, closed connections, 5xx responses and 429 are transient NCBI - * failures worth retrying. Other 4xx errors are permanent. - */ - private static boolean isRetryable(IOException e) - { - if (e instanceof SocketTimeoutException || e instanceof SocketException || e instanceof NoHttpResponseException) - { - return true; - } - if (e instanceof HttpResponseException hre) - { - return hre.getStatusCode() >= 500 || hre.getStatusCode() == TOO_MANY_REQUESTS; - } - return false; - } - - // 500ms after the first failure, then doubling. - private static long retryDelayMs(int attempt) - { - return (long) RETRY_BASE_DELAY_MS << (attempt - 1); - } - - /** - * @return false if the thread was interrupted, in which case the caller should stop rather than - * carry on without the delay it asked for. - */ - protected boolean sleepMs(long ms) - { - try - { - Thread.sleep(ms); - return true; - } - catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - return false; - } + return _httpClient.getString(url, getLog(log)); } /** @@ -1162,7 +980,7 @@ static List extractTitleKeywords(String title) */ private void rateLimit() { - sleepMs(RATE_LIMIT_DELAY_MS); + _httpClient.sleepMs(RATE_LIMIT_DELAY_MS); } /** @@ -1670,89 +1488,7 @@ public void testPublicationMatchRoundTrip() assertFalse(restored.matchesProteomeXchangeId()); } - // -- HTTP retry tests -- - - @Test - public void testIsRetryable() - { - // Read timeouts and 5xx responses are transient NCBI failures -> retry - assertTrue(isRetryable(new SocketTimeoutException("Read timed out"))); - assertTrue(isRetryable(new HttpResponseException(500, "Internal Server Error"))); - assertTrue(isRetryable(new HttpResponseException(503, "Service Unavailable"))); - - // A connection reset or closed by the server is transient -> retry - assertTrue("A connection reset should be retried", isRetryable(new SocketException("Connection reset"))); - assertTrue("A connection closed without a response should be retried", - isRetryable(new NoHttpResponseException("The target server failed to respond"))); - - // 429 is NCBI's response when the request rate is exceeded, which backoff is for - assertTrue(isRetryable(new HttpResponseException(429, "Too Many Requests"))); - - // Other 4xx and generic IO errors are permanent -> fail fast - assertFalse(isRetryable(new HttpResponseException(400, "Bad Request"))); - assertFalse(isRetryable(new HttpResponseException(404, "Not Found"))); - assertFalse(isRetryable(new IOException("Stream closed"))); - } - - @Test - public void testRetryDelayMs() - { - // Exponential backoff of 500ms then 1000ms. With MAX_HTTP_ATTEMPTS at 3 the loop sleeps - // after the first two failures and rethrows after the third, so those are the only - // delays a request can wait. - assertEquals(500, retryDelayMs(1)); - assertEquals(1000, retryDelayMs(2)); - assertEquals("A request sleeps once per failed attempt except the last, so only the" - + " delays asserted above are reachable", 2, MAX_HTTP_ATTEMPTS - 1); - } - - @Test - public void testErrorDetail() - { - // 4xx: the response body is appended so the cause (e.g. an invalid API key) is logged - String detail = errorDetail(400, "Bad Request", "{\"error\":\"API key invalid\"}", null); - assertTrue(detail.contains("Bad Request")); - assertTrue(detail.contains("API key invalid")); - - // 5xx: body omitted (uninformative) - assertEquals("Internal Server Error", errorDetail(500, "Internal Server Error", "oops", null)); - - // 4xx with blank or null body: just the reason phrase, no trailing separator - assertEquals("Bad Request", errorDetail(400, "Bad Request", "", null)); - assertEquals("Bad Request", errorDetail(400, "Bad Request", null, null)); - - // A body that quotes the request back must not carry the key into the log - String echoed = errorDetail(400, "Bad Request", "invalid key SECRET123 for api_key=SECRET123", "SECRET123"); - assertFalse("errorDetail must not put the API key in the message", echoed.contains("SECRET123")); - - // A key that spans the truncation point must not leave its prefix in the message - String padding = "x".repeat(MAX_ERROR_BODY_CHARS - 10); - String spanning = errorDetail(400, "Bad Request", padding + " SECRET123456789 trailing", "SECRET123456789"); - assertFalse("errorDetail must not put part of the API key in the message", spanning.contains("SECRET")); - } - - @Test - public void testRedactApiKey() - { - // The key is stripped from an eutils URL, and the rest of the URL is left intact - String url = "https://eutils.ncbi.nlm.nih.gov/esearch.fcgi?db=pmc&api_key=SECRET123&term=PXD001"; - String redacted = redactApiKey(url, "SECRET123"); - assertFalse("The redacted URL must not contain the key", redacted.contains("SECRET123")); - assertTrue("The redacted URL must keep its other parameters", redacted.contains("term=PXD001")); - assertTrue(redacted.contains("db=pmc")); - - // The key is stripped even when it appears without the api_key= prefix - assertFalse(redactApiKey("rejected key SECRET123", "SECRET123").contains("SECRET123")); - - // A URL with no key is unchanged, and null text stays null - String noKey = "https://eutils.ncbi.nlm.nih.gov/esearch.fcgi?db=pmc&term=PXD001"; - assertEquals(noKey, redactApiKey(noKey, null)); - assertNull(redactApiKey(null, "SECRET123")); - - // The key is recovered from the URL so callers do not have to read the settings - assertEquals("SECRET123", apiKeyFrom(url)); - assertNull(apiKeyFrom(noKey)); - } + // -- Request parameter and API key check tests -- @Test public void testBuildCommonParams() @@ -1773,42 +1509,6 @@ public void testBuildCommonParams() assertTrue(buildCommonParams("pmc", " ABC123 ").contains("api_key=ABC123")); } - @Test - public void testGetStringRetriesTransientFailures() throws IOException - { - // executeGet returns a 5xx twice, then succeeds. getString should retry and return the body. - int[] attempts = {0}; - NoWaitService service = new NoWaitService() - { - @Override - protected String executeGet(String url) throws IOException - { - if (++attempts[0] < 3) - throw new HttpResponseException(503, "Service Unavailable"); - return "body"; - } - }; - assertEquals("body", service.getString("http://test", LOG)); - assertEquals("Should retry until the 3rd attempt succeeds", 3, attempts[0]); - assertEquals("The loop should wait 500ms then 1000ms", List.of(500L, 1000L), service.sleeps); - } - - /** - * Runs the retry loop without waiting, and records the delays it asked for. A test can then - * check the delays the loop used, which retryDelayMs on its own cannot show. - */ - private static class NoWaitService extends NcbiPublicationSearchServiceImpl - { - private final List sleeps = new ArrayList<>(); - - @Override - protected boolean sleepMs(long ms) - { - sleeps.add(ms); - return true; - } - } - @Test public void testCheckApiKey() { @@ -1841,7 +1541,7 @@ private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure) private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure, String apiKey) { - NcbiPublicationSearchServiceImpl service = new NoWaitService() + NcbiHttpClient client = new NcbiHttpClient.TestCase.NoWaitClient() { @Override protected String executeGet(String url) throws IOException @@ -1853,102 +1553,24 @@ protected String executeGet(String url) throws IOException return "{\"esearchresult\":{\"idlist\":[]}}"; } }; - return service.checkApiKey(apiKey); - } - - @Test - public void testGetStringStopsWhenInterrupted() - { - // An interrupted thread cannot wait, so retrying would send the remaining attempts back - // to back. getString should give up after the first failure instead. - int[] attempts = {0}; - NcbiPublicationSearchServiceImpl service = new NcbiPublicationSearchServiceImpl() - { - @Override - protected String executeGet(String url) throws IOException - { - attempts[0]++; - Thread.currentThread().interrupt(); - throw new HttpResponseException(503, "Service Unavailable"); - } - }; - - try - { - service.getString("http://test", LOG); - fail("Expected the interrupted request to be rethrown"); - } - catch (IOException expected) - { - assertEquals("An interrupted request should not be retried", 1, attempts[0]); - } - finally - { - // Clear the flag so it cannot reach whatever test runs next. - Thread.interrupted(); - } - } - - @Test - public void testGetStringGivesUpAfterMaxAttempts() - { - // executeGet always returns a 5xx. getString should try MAX_HTTP_ATTEMPTS times, then rethrow. - int[] attempts = {0}; - NoWaitService service = new NoWaitService() - { - @Override - protected String executeGet(String url) throws IOException - { - attempts[0]++; - throw new HttpResponseException(500, "Internal Server Error"); - } - }; - try - { - service.getString("http://test", LOG); - fail("Expected HttpResponseException after exhausting retries"); - } - catch (HttpResponseException e) - { - assertEquals(500, e.getStatusCode()); - } - catch (IOException e) - { - fail("Expected HttpResponseException, got " + e); - } - assertEquals("Should attempt exactly MAX_HTTP_ATTEMPTS times", MAX_HTTP_ATTEMPTS, attempts[0]); - assertEquals("One wait fewer than attempts, and no wait after the last", List.of(500L, 1000L), service.sleeps); + return new NcbiPublicationSearchServiceImpl(client).checkApiKey(apiKey); } @Test - public void testGetStringDoesNotRetryClientErrors() - { - // A 4xx is permanent. getString should fail immediately without retrying. - int[] attempts = {0}; - NoWaitService service = new NoWaitService() - { - @Override - protected String executeGet(String url) throws IOException - { - attempts[0]++; - throw new HttpResponseException(400, "Bad Request"); - } - }; - try - { - service.getString("http://test", LOG); - fail("Expected HttpResponseException for a 4xx"); - } - catch (HttpResponseException e) - { - assertEquals(400, e.getStatusCode()); - } - catch (IOException e) - { - fail("Expected HttpResponseException, got " + e); - } - assertEquals("4xx must not be retried", 1, attempts[0]); - assertTrue("A 4xx must not wait", service.sleeps.isEmpty()); + public void testMockResponses() + { + // The Selenium tests use the mock only on TeamCity. Requests from the service should reach the + // mock's canned responses through NcbiHttpClient.getString. + MockNcbiPublicationSearchService mock = new MockNcbiPublicationSearchService(); + String citation = "Smith J. A test title. J Proteome Res. 2024."; + mock.register("pubmed", "12345", "Smith", null, "A test title", "Smith J", + "2024/01/15 00:00", "J Proteome Res", "Journal of Proteome Research", citation); + + assertEquals("The mock should return the registered citation", citation, mock.getCitation("12345", DB.PubMed)); + assertNull("The mock should return no citation for an ID that was not registered", + mock.getCitation("67890", DB.PubMed)); + assertEquals("The mock should report any API key as valid", NcbiApiKeyCheck.Status.VALID, + mock.checkApiKey("test-key").getStatus()); } // -- Helper methods for building test JSON -- From abfa8b02d1a18aa649564876cc43b59de8d71988 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 27 Sep 2026 15:14:31 -0700 Subject: [PATCH 21/24] Addressed code review findings on the NCBI API key and NcbiHttpClient * PrivateDataReminderSettings reads no key when the saved key cannot be decrypted, for example after the encryption key changed. saveNcbiApiKey deletes such a key and saves the new one * ValidateNcbiApiKeyAction responds with the configuration message when no encryption key is configured. The message now says a key cannot be saved or removed * checkApiKey takes the caller's logger, so the reminder job's key check retry warnings go to the job log * testCheckApiKey counts requests, so it fails if the service stops going through the retry loop * NcbiHttpClient has no logger of its own. Its tests use a test logger * PanoramaPublicBaseTest.getPrivateDataReminderSettings reports no saved key when the key field is not displayed Co-Authored-By: Claude --- .../PanoramaPublicController.java | 9 +++++- .../message/PrivateDataReminderSettings.java | 31 ++++++++++++++----- .../panoramapublic/ncbi/NcbiHttpClient.java | 4 +-- .../ncbi/NcbiPublicationSearchService.java | 5 +-- .../NcbiPublicationSearchServiceImpl.java | 22 ++++++++++--- .../pipeline/PrivateDataReminderJob.java | 2 +- .../PanoramaPublicBaseTest.java | 6 ++-- 7 files changed, 60 insertions(+), 19 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java index d5217f2c..3343096a 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java @@ -10095,6 +10095,13 @@ public Object execute(PrivateDataReminderSettingsForm form, BindException errors ApiSimpleResponse response = new ApiSimpleResponse(); response.put("success", true); + if (!Encryption.isEncryptionPassPhraseSpecified()) + { + response.put("valid", false); + response.put("message", PrivateDataReminderSettings.NCBI_API_KEY_REQUIRES_ENCRYPTION); + return response; + } + // An empty field means check the key that is already saved. boolean checkingSavedKey = StringUtils.isBlank(form.getNcbiApiKey()); String apiKey = checkingSavedKey @@ -10108,7 +10115,7 @@ public Object execute(PrivateDataReminderSettingsForm form, BindException errors return response; } - NcbiApiKeyCheck check = NcbiPublicationSearchService.get().checkApiKey(apiKey); + NcbiApiKeyCheck check = NcbiPublicationSearchService.get().checkApiKey(apiKey, LOG); response.put("valid", check.isValid()); if (check.isValid()) { diff --git a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java index c04892f2..955e017d 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java +++ b/panoramapublic/src/org/labkey/panoramapublic/message/PrivateDataReminderSettings.java @@ -46,8 +46,8 @@ public class PrivateDataReminderSettings public static final String PROP_PUBLICATION_SEARCH_FREQUENCY = "Publication search frequency (months)"; public static final String PROP_NCBI_API_KEY = "NCBI API key"; public static final String PROP_NCBI_CREDENTIALS = "Panorama Public NCBI credentials"; - public static final String NCBI_API_KEY_REQUIRES_ENCRYPTION = "An NCBI API key cannot be saved because this server" - + " has no encryption key configured."; + public static final String NCBI_API_KEY_REQUIRES_ENCRYPTION = "An NCBI API key cannot be saved or removed because" + + " this server has no encryption key configured."; private static final boolean DEFAULT_ENABLE_REMINDERS = false; public static final String DEFAULT_REMINDER_TIME = "8:00 AM"; @@ -166,8 +166,17 @@ public static void save(PrivateDataReminderSettings settings) */ public static void saveNcbiApiKey(@Nullable String apiKey) { - PropertyManager.WritablePropertyMap credentials = - PropertyManager.getEncryptedStore().getWritableProperties(PROP_NCBI_CREDENTIALS, true); + PropertyManager.WritablePropertyMap credentials; + try + { + credentials = PropertyManager.getEncryptedStore().getWritableProperties(PROP_NCBI_CREDENTIALS, true); + } + catch (Encryption.DecryptionException e) + { + // The saved key cannot be decrypted, for example after the encryption key changed. Delete it and start over. + PropertyManager.getEncryptedStore().deletePropertySet(PROP_NCBI_CREDENTIALS); + credentials = PropertyManager.getEncryptedStore().getWritableProperties(PROP_NCBI_CREDENTIALS, true); + } if (StringUtils.isBlank(apiKey)) { credentials.remove(PROP_NCBI_API_KEY); @@ -191,9 +200,17 @@ public static boolean hasNcbiApiKey() { return null; } - Map credentials = - PropertyManager.getEncryptedStore().getProperties(PROP_NCBI_CREDENTIALS); - return credentials.get(PROP_NCBI_API_KEY); + try + { + Map credentials = + PropertyManager.getEncryptedStore().getProperties(PROP_NCBI_CREDENTIALS); + return credentials.get(PROP_NCBI_API_KEY); + } + catch (Encryption.DecryptionException e) + { + // The key was saved under a different encryption key. Run without it until an admin saves a new one. + return null; + } } public void setEnableReminders(boolean enableReminders) diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiHttpClient.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiHttpClient.java index 49ea1d5c..d2cc3950 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiHttpClient.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiHttpClient.java @@ -49,8 +49,6 @@ */ public class NcbiHttpClient { - private static final Logger LOG = LogHelper.getLogger(NcbiHttpClient.class, "HTTP requests to NCBI for the Panorama Public publication search"); - private static final int TIMEOUT_MS = 10000; // 10 seconds // NCBI eutils fails intermittently, even well under the rate limit. Retry the transient @@ -242,6 +240,8 @@ protected boolean sleepMs(long ms) public static class TestCase extends Assert { + private static final Logger LOG = LogHelper.getLogger(TestCase.class, "NcbiHttpClient tests"); + @Test public void testIsRetryable() { diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java index d1056ffb..ba33906b 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchService.java @@ -39,9 +39,10 @@ static NcbiPublicationSearchService get() @Nullable String getCitation(String publicationId, DB database); /** - * Send a minimal request to NCBI with the given key. + * Send a minimal request to NCBI with the given key. Retry warnings are written to {@code logger}, or to + * the service's own logger if it is null. */ - @NotNull NcbiApiKeyCheck checkApiKey(@Nullable String apiKey); + @NotNull NcbiApiKeyCheck checkApiKey(@Nullable String apiKey, @Nullable Logger logger); @Nullable Pair getPubMedLinkAndCitation(String pubmedId); diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index fe3d0616..5fddc8dd 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -389,13 +389,13 @@ private List executeSearch(String query, String database, Logger log) } @Override - public @NotNull NcbiApiKeyCheck checkApiKey(@Nullable String apiKey) + public @NotNull NcbiApiKeyCheck checkApiKey(@Nullable String apiKey, @Nullable Logger logger) { // A minimal ESearch request. getString retries a transient failure before the check gives up. String url = ESEARCH_URL + "?" + buildCommonParams("pubmed", apiKey) + "&term=labkey&retmax=1&retmode=json"; try { - getString(url, LOG); + getString(url, logger); return NcbiApiKeyCheck.valid(); } catch (HttpResponseException e) @@ -1532,6 +1532,14 @@ public void testCheckApiKey() NcbiApiKeyCheck rejected = checkApiKeyAgainst( new HttpResponseException(400, "Bad Request - invalid key SECRET123"), "SECRET123"); assertFalse("A rejection must not carry the key", rejected.getMessage().contains("SECRET123")); + + // The service's requests go through the retry loop in NcbiHttpClient.getString + int[] attempts = {0}; + checkApiKeyAgainst(new HttpResponseException(503, "Service Unavailable"), "test-key", attempts); + assertEquals("A 503 from the key check should be tried 3 times", 3, attempts[0]); + attempts[0] = 0; + checkApiKeyAgainst(new HttpResponseException(400, "Bad Request"), "test-key", attempts); + assertEquals("A 400 from the key check should be tried once", 1, attempts[0]); } private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure) @@ -1540,12 +1548,18 @@ private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure) } private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure, String apiKey) + { + return checkApiKeyAgainst(failure, apiKey, new int[1]); + } + + private NcbiApiKeyCheck checkApiKeyAgainst(IOException failure, String apiKey, int[] attempts) { NcbiHttpClient client = new NcbiHttpClient.TestCase.NoWaitClient() { @Override protected String executeGet(String url) throws IOException { + attempts[0]++; if (failure != null) { throw failure; @@ -1553,7 +1567,7 @@ protected String executeGet(String url) throws IOException return "{\"esearchresult\":{\"idlist\":[]}}"; } }; - return new NcbiPublicationSearchServiceImpl(client).checkApiKey(apiKey); + return new NcbiPublicationSearchServiceImpl(client).checkApiKey(apiKey, null); } @Test @@ -1570,7 +1584,7 @@ public void testMockResponses() assertNull("The mock should return no citation for an ID that was not registered", mock.getCitation("67890", DB.PubMed)); assertEquals("The mock should report any API key as valid", NcbiApiKeyCheck.Status.VALID, - mock.checkApiKey("test-key").getStatus()); + mock.checkApiKey("test-key", null).getStatus()); } // -- Helper methods for building test JSON -- diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index 397b82ac..40c23aba 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -351,7 +351,7 @@ private boolean ncbiApiKeyAccepted() return true; } - NcbiApiKeyCheck check = NcbiPublicationSearchService.get().checkApiKey(apiKey); + NcbiApiKeyCheck check = NcbiPublicationSearchService.get().checkApiKey(apiKey, getLogger()); if (check.isRejected()) { getLogger().error("NCBI rejected the API key, so no reminders were posted. Correct the key on the Private Data Reminder Settings page and run the job again. {}", diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index 44a39e68..6a98b5e1 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -647,8 +647,10 @@ protected Map getPrivateDataReminderSettings() settings.put("enablePublicationSearch", String.valueOf(Locator.checkboxByName("enablePublicationSearch").findElement(getDriver()).isSelected())); settings.put("publicationSearchFrequency", getFormElement(Locator.input("publicationSearchFrequency"))); // The saved key is never displayed. data-key-saved on the field reports whether one is stored. - settings.put("ncbiApiKeySaved", - Locator.input("ncbiApiKey").findElement(getDriver()).getDomAttribute("data-key-saved")); + // The field is not rendered on a server with no encryption key, where no key can be saved. + settings.put("ncbiApiKeySaved", Locator.input("ncbiApiKey").findOptionalElement(getDriver()) + .map(field -> field.getDomAttribute("data-key-saved")) + .orElse("false")); return settings; } From 3d30a1f142f3622893f0105ab5eb2d8510baa42d Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 4 Oct 2026 18:15:13 -0700 Subject: [PATCH 22/24] Added an NCBI check of the API key before it is saved * PrivateDataReminderSettingsAction.validateCommand checks a new key with NCBI and saves it only if NCBI accepts it. A rejected key and a check that could not be completed each display a reason on the form and log a WARN * MockNcbiPublicationSearchService responds to two designated keys with a 400 and a 503 * NcbiApiKeyTest uses the mock on every server and checks that neither key is saved Co-Authored-By: Claude --- .../PanoramaPublicController.java | 17 ++++++ .../MockNcbiPublicationSearchService.java | 18 +++++- .../tests/panoramapublic/NcbiApiKeyTest.java | 61 ++++++++++--------- 3 files changed, 65 insertions(+), 31 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java index 3343096a..6ef71cd5 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java @@ -10196,6 +10196,23 @@ else if (PrivateDataReminderSettings.parseReminderTime(form.getReminderTime()) = { errors.reject(ERROR_MSG, PrivateDataReminderSettings.NCBI_API_KEY_REQUIRES_ENCRYPTION); } + // Save a new key only if NCBI accepts it. + if (!errors.hasErrors() && !form.isClearNcbiApiKey() && !StringUtils.isBlank(form.getNcbiApiKey())) + { + NcbiApiKeyCheck check = NcbiPublicationSearchService.get().checkApiKey(form.getNcbiApiKey().trim(), LOG); + String detail = StringUtils.defaultString(check.getMessage()); + if (check.isRejected()) + { + LOG.warn("NCBI rejected an API key entered on the Private Data Reminder Settings page. {}", detail); + errors.reject(ERROR_MSG, "NCBI rejected this API key, so it was not saved. " + detail); + } + else if (!check.isValid()) + { + LOG.warn("Could not check an API key entered on the Private Data Reminder Settings page. {}", detail); + errors.reject(ERROR_MSG, "Could not check this API key with NCBI, so it was not saved." + + " Try again later. " + detail); + } + } } @Override diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java index 9de89510..63cbef5b 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/MockNcbiPublicationSearchService.java @@ -15,6 +15,7 @@ */ package org.labkey.panoramapublic.ncbi; +import org.apache.hc.client5.http.HttpResponseException; import org.jetbrains.annotations.Nullable; import org.json.JSONArray; import org.json.JSONObject; @@ -30,7 +31,7 @@ /** * Mock implementation of {@link NcbiPublicationSearchService} that returns canned data registered by tests. - * Used by Selenium tests when running on TeamCity. + * Used by NcbiApiKeyTest on every server, and by PublicationSearchTest on TeamCity. * Extends {@link NcbiPublicationSearchServiceImpl} and gives it an {@link NcbiHttpClient} whose * {@code executeGet()} returns canned responses in place of the real HTTP request. The retry loop in * {@link NcbiHttpClient#getString}, and all search logic, filtering, author/title verification, citation @@ -41,6 +42,13 @@ */ public class MockNcbiPublicationSearchService extends NcbiPublicationSearchServiceImpl { + // The mock responds to a request carrying this API key with a 400, which is NCBI's response to a key it + // does not recognize. NcbiApiKeyTest uses the same value. + public static final String REJECTED_API_KEY = "mock-rejected-ncbi-api-key"; + // The mock responds to a request carrying this API key with a 503, so the key check cannot be completed. + // NcbiApiKeyTest uses the same value. + public static final String UNCHECKED_API_KEY = "mock-unchecked-ncbi-api-key"; + private final CannedResponses _responses; public MockNcbiPublicationSearchService() @@ -153,6 +161,14 @@ private static class CannedResponses extends NcbiHttpClient @Override protected String executeGet(String url) throws IOException { + if (REJECTED_API_KEY.equals(apiKeyFrom(url))) + { + throw new HttpResponseException(400, "Bad Request - API key invalid"); + } + if (UNCHECKED_API_KEY.equals(apiKeyFrom(url))) + { + throw new HttpResponseException(503, "Service Unavailable"); + } if (url.contains("esearch.fcgi")) { return handleESearch(url).toString(); diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java index c471ec9e..24233365 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java @@ -22,7 +22,6 @@ import org.labkey.remoteapi.SimplePostCommand; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; -import org.labkey.test.TestProperties; import org.labkey.test.categories.External; import org.labkey.test.categories.MacCossLabModules; @@ -30,7 +29,6 @@ import java.util.Map; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; /** @@ -46,6 +44,10 @@ public class NcbiApiKeyTest extends PanoramaPublicBaseTest { private static final String TEST_API_KEY = "test-ncbi-api-key"; + // MockNcbiPublicationSearchService.REJECTED_API_KEY. The mock responds to it with a 400. + private static final String REJECTED_API_KEY = "mock-rejected-ncbi-api-key"; + // MockNcbiPublicationSearchService.UNCHECKED_API_KEY. The mock responds to it with a 503. + private static final String UNCHECKED_API_KEY = "mock-unchecked-ncbi-api-key"; private boolean _savedTestApiKey = false; private boolean _useMockNcbi = false; @@ -65,6 +67,9 @@ public void testNcbiApiKeySettings() + " Private Data Reminder Settings page, run this test, then enter the key again.", "false", _originalReminderSettings.get("ncbiApiKeySaved")); + verifyKeyNotSaved(REJECTED_API_KEY, "NCBI rejected this API key, so it was not saved."); + verifyKeyNotSaved(UNCHECKED_API_KEY, "Could not check this API key with NCBI, so it was not saved."); + _savedTestApiKey = true; savePrivateDataReminderSettings("2", "0", "0", true, TEST_API_KEY); @@ -90,48 +95,44 @@ public void testNcbiApiKeySettings() } /** - * Covers the button, the request it sends and the message it displays. The mock responds to every - * request, so it reports the key as accepted. + * Save checks a new key with NCBI. A key NCBI has not accepted should not be saved, and the form should + * display the reason. + */ + private void verifyKeyNotSaved(String apiKey, String expectedMessage) + { + getPrivateDataReminderSettings(); + setFormElement(Locator.input("ncbiApiKey"), apiKey); + clickButton("Save"); + assertTextPresent(expectedMessage); + assertEquals("A key NCBI has not accepted should not be saved", "false", + getPrivateDataReminderSettings().get("ncbiApiKeySaved")); + } + + /** + * Covers the button, the request it sends and the message it displays for a key the mock accepts and + * a key it rejects. */ private void verifyValidateButton() { + Locator result = Locator.id("ncbiApiKeyValidationResult"); + setFormElement(Locator.input("ncbiApiKey"), "not-a-real-key"); click(Locator.lkButton("Validate")); + waitForElement(result.containing("NCBI accepted this key")); - Locator result = Locator.id("ncbiApiKeyValidationResult"); - if (_useMockNcbi) - { - waitForElement(result.containing("NCBI accepted this key")); - } - else - { - // NCBI rejects this key with a 400, or returns a 5xx and the check is unconfirmed. - // Neither reports the key as accepted. The retries mean a live check can take half a minute. - waitFor(() -> { - String text = result.findElement(getDriver()).getText(); - return !text.isEmpty() && !text.startsWith("Checking"); - }, - "NCBI validation result was not displayed", WAIT_FOR_PAGE); - - String message = result.findElement(getDriver()).getText(); - assertFalse("A key NCBI does not recognize must not be reported as accepted. Message: " + message, - message.contains("accepted")); - } + setFormElement(Locator.input("ncbiApiKey"), REJECTED_API_KEY); + click(Locator.lkButton("Validate")); + waitForElement(result.containing("NCBI rejected this key")); setFormElement(Locator.input("ncbiApiKey"), ""); } /* - * On TeamCity, route NCBI requests through the mock so this test does not depend on NCBI being - * reachable. On a development machine, use the real service so a key can actually be rejected. + * Route NCBI requests through the mock, so the test does not depend on NCBI being reachable and a key + * can be rejected on demand. */ private void setupMockNcbiService() { - if (!TestProperties.isTestRunningOnTeamCity()) - { - return; - } - try { SimplePostCommand command = new SimplePostCommand("panoramapublic", "setupMockNcbiService"); From 6ccc96dff045384e4cd5845dc093d9b769b2f930 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 4 Oct 2026 20:04:13 -0700 Subject: [PATCH 23/24] Changed the reminder settings test cleanup to restore settings through an API call * Added PanoramaPublicController.RestorePrivateDataReminderSettingsAction, a dev-mode test support action that sets the reminder settings passed in and can remove a saved NCBI API key * NcbiApiKeyTest and PublicationSearchTest restore settings in @After through the action, so the cleanup loads no page and the failure screenshot shows the page that failed * NcbiApiKeyTest.restoreAfterTest sets clearNcbiApiKey when the settings captured at the start of the test report no saved key, replacing the _savedTestApiKey flag. The action removes the key only if PrivateDataReminderSettings.hasNcbiApiKey() returns true * PrivateDataReminderTest restores the reminder settings it changes, which it did not before * Renamed requireDevModeForMockNcbiService to requireDevModeForTestSupport Co-Authored-By: Claude --- .../PanoramaPublicController.java | 84 +++++++++++++++++-- .../tests/panoramapublic/NcbiApiKeyTest.java | 27 ++---- .../PanoramaPublicBaseTest.java | 28 +++++++ .../PrivateDataReminderTest.java | 15 ++++ .../panoramapublic/PublicationSearchTest.java | 7 +- 5 files changed, 128 insertions(+), 33 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java index 6ef71cd5..f8ce5122 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java @@ -11350,12 +11350,13 @@ public static ActionURL getCatalogImageDownloadUrl(ExperimentAnnotations expAnno // ======================== Support actions for Selenium tests ======================== // These actions swap the process-wide NCBI publication search service to a mock so that Selenium tests - // run without calling the live NCBI API. They must never be reachable on a production server (non-dev-mode) - private static void requireDevModeForMockNcbiService() + // run without calling the live NCBI API, and restore the settings a test changed. They must never be + // reachable on a production server (non-dev-mode) + private static void requireDevModeForTestSupport() { if (!AppProps.getInstance().isDevMode()) { - throw new NotFoundException("Mock NCBI publication search service actions are only available on a server running in dev mode."); + throw new NotFoundException("Selenium test support actions are only available on a server running in dev mode."); } } @@ -11365,7 +11366,7 @@ public static class SetupMockNcbiServiceAction extends MutatingApiAction @Override public Object execute(Object form, BindException errors) { - requireDevModeForMockNcbiService(); + requireDevModeForTestSupport(); NcbiPublicationSearchServiceImpl.setInstance(new MockNcbiPublicationSearchService()); return new ApiSimpleResponse("mock", true); } @@ -11377,7 +11378,7 @@ public static class RestoreNcbiServiceAction extends MutatingApiAction @Override public Object execute(Object form, BindException errors) { - requireDevModeForMockNcbiService(); + requireDevModeForTestSupport(); NcbiPublicationSearchServiceImpl.setInstance(new NcbiPublicationSearchServiceImpl()); return new ApiSimpleResponse("restored", true); } @@ -11389,7 +11390,7 @@ public static class RegisterMockPublicationAction extends MutatingApiAction + { + @Override + public Object execute(RestorePrivateDataReminderSettingsForm form, BindException errors) + { + requireDevModeForTestSupport(); + PrivateDataReminderSettings settings = PrivateDataReminderSettings.get(); + if (form.getExtensionLength() != null) + { + settings.setExtensionLength(form.getExtensionLength()); + } + if (form.getDelayUntilFirstReminder() != null) + { + settings.setDelayUntilFirstReminder(form.getDelayUntilFirstReminder()); + } + if (form.getReminderFrequency() != null) + { + settings.setReminderFrequency(form.getReminderFrequency()); + } + if (form.getEnablePublicationSearch() != null) + { + settings.setEnablePublicationSearch(form.getEnablePublicationSearch()); + } + if (form.getPublicationSearchFrequency() != null) + { + settings.setPublicationSearchFrequency(form.getPublicationSearchFrequency()); + } + PrivateDataReminderSettings.save(settings); + // hasNcbiApiKey is false on a server with no encryption key, where saveNcbiApiKey throws. + if (form.isClearNcbiApiKey() && PrivateDataReminderSettings.hasNcbiApiKey()) + { + PrivateDataReminderSettings.saveNcbiApiKey(null); + } + return new ApiSimpleResponse("restored", true); + } + } + + public static class RestorePrivateDataReminderSettingsForm + { + private Integer _extensionLength; + private Integer _delayUntilFirstReminder; + private Integer _reminderFrequency; + private Boolean _enablePublicationSearch; + private Integer _publicationSearchFrequency; + private boolean _clearNcbiApiKey; + + public Integer getExtensionLength() { return _extensionLength; } + public void setExtensionLength(Integer extensionLength) { _extensionLength = extensionLength; } + + public Integer getDelayUntilFirstReminder() { return _delayUntilFirstReminder; } + public void setDelayUntilFirstReminder(Integer delayUntilFirstReminder) { _delayUntilFirstReminder = delayUntilFirstReminder; } + + public Integer getReminderFrequency() { return _reminderFrequency; } + public void setReminderFrequency(Integer reminderFrequency) { _reminderFrequency = reminderFrequency; } + + public Boolean getEnablePublicationSearch() { return _enablePublicationSearch; } + public void setEnablePublicationSearch(Boolean enablePublicationSearch) { _enablePublicationSearch = enablePublicationSearch; } + + public Integer getPublicationSearchFrequency() { return _publicationSearchFrequency; } + public void setPublicationSearchFrequency(Integer publicationSearchFrequency) { _publicationSearchFrequency = publicationSearchFrequency; } + + public boolean isClearNcbiApiKey() { return _clearNcbiApiKey; } + public void setClearNcbiApiKey(boolean clearNcbiApiKey) { _clearNcbiApiKey = clearNcbiApiKey; } + } + public static class TestCase extends AbstractActionPermissionTest { @Override diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java index 24233365..115dcd59 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/NcbiApiKeyTest.java @@ -49,7 +49,6 @@ public class NcbiApiKeyTest extends PanoramaPublicBaseTest // MockNcbiPublicationSearchService.UNCHECKED_API_KEY. The mock responds to it with a 503. private static final String UNCHECKED_API_KEY = "mock-unchecked-ncbi-api-key"; - private boolean _savedTestApiKey = false; private boolean _useMockNcbi = false; private Map _originalReminderSettings; @@ -58,7 +57,7 @@ public void testNcbiApiKeySettings() { setupMockNcbiService(); - // Capture the existing reminder settings up front so removeTestApiKey can put them back. A dev + // Capture the existing reminder settings up front so restoreAfterTest can put them back. A dev // machine may have non-default values set. _originalReminderSettings = getPrivateDataReminderSettings(); @@ -70,7 +69,6 @@ public void testNcbiApiKeySettings() verifyKeyNotSaved(REJECTED_API_KEY, "NCBI rejected this API key, so it was not saved."); verifyKeyNotSaved(UNCHECKED_API_KEY, "Could not check this API key with NCBI, so it was not saved."); - _savedTestApiKey = true; savePrivateDataReminderSettings("2", "0", "0", true, TEST_API_KEY); assertEquals("The saved key must never be rendered into the form", "", @@ -87,7 +85,6 @@ public void testNcbiApiKeySettings() // Removing a key takes the explicit checkbox. checkCheckbox(Locator.checkboxByName("clearNcbiApiKey")); clickButton("Save"); - _savedTestApiKey = false; assertEquals("Remove the saved key should remove it", "false", getPrivateDataReminderSettings().get("ncbiApiKeySaved")); assertElementNotPresent("Remove the saved key should be offered only when a key is saved", @@ -161,31 +158,19 @@ private void restoreNcbiService() } @After - public void removeTestApiKey() + public void restoreAfterTest() { if (_useMockNcbi) { restoreNcbiService(); } - if (_savedTestApiKey) - { - // Remove the key even when the test failed before its own removal step. A key NCBI - // rejects makes every publication search on this server fail, including the next run's. - getPrivateDataReminderSettings(); - checkCheckbox(Locator.checkboxByName("clearNcbiApiKey")); - clickButton("Save"); - } - if (_originalReminderSettings != null) { - // The reminder settings are site wide. Restore the values this test overwrote. - savePrivateDataReminderSettings( - _originalReminderSettings.get("extensionLength"), - _originalReminderSettings.get("delayUntilFirstReminder"), - _originalReminderSettings.get("reminderFrequency"), - Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch")), - null); + // The reminder settings are site wide. Restore the values this test overwrote, and remove any key the + // test saved. A key that was saved before the test cannot be read back, so it is left in place. + restorePrivateDataReminderSettings(_originalReminderSettings, + "false".equals(_originalReminderSettings.get("ncbiApiKeySaved"))); } } } diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index 6a98b5e1..1220dd69 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -27,6 +27,7 @@ import org.labkey.remoteapi.CommandResponse; import org.labkey.remoteapi.Connection; import org.labkey.remoteapi.SimpleGetCommand; +import org.labkey.remoteapi.SimplePostCommand; import org.labkey.test.Locator; import org.labkey.test.TestFileUtils; import org.labkey.test.TestTimeoutException; @@ -706,6 +707,33 @@ protected void savePrivateDataReminderSettings(String extensionLength, String de } } + /** + * Restores the settings read by getPrivateDataReminderSettings through an API call, which loads no page. For + * use in an @After method, where loading a page would replace the failed page in the failure screenshot. + * @param clearNcbiApiKey true to remove the saved NCBI API key + */ + protected void restorePrivateDataReminderSettings(Map original, boolean clearNcbiApiKey) + { + Map params = new HashMap<>(); + params.put("extensionLength", original.get("extensionLength")); + params.put("delayUntilFirstReminder", original.get("delayUntilFirstReminder")); + params.put("reminderFrequency", original.get("reminderFrequency")); + params.put("enablePublicationSearch", original.get("enablePublicationSearch")); + params.put("publicationSearchFrequency", original.get("publicationSearchFrequency")); + params.put("clearNcbiApiKey", clearNcbiApiKey); + + SimplePostCommand command = new SimplePostCommand("panoramapublic", "restorePrivateDataReminderSettings"); + command.setParameters(params); + try + { + command.execute(createDefaultConnection(), "/"); + } + catch (IOException | CommandException e) + { + throw new RuntimeException("Failed to restore the Private Data Reminder Settings", e); + } + } + protected void goToSendRemindersPage(String projectName) { goToAdminConsole().goToSettingsSection(); diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PrivateDataReminderTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PrivateDataReminderTest.java index d677ffdd..326fa588 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PrivateDataReminderTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PrivateDataReminderTest.java @@ -15,6 +15,7 @@ */ package org.labkey.test.tests.panoramapublic; +import org.junit.After; import org.junit.Test; import org.junit.experimental.categories.Category; import org.labkey.test.BaseWebDriverTest; @@ -27,6 +28,7 @@ import org.labkey.test.util.DataRegionTable; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.labkey.test.util.PermissionsHelper.READER_ROLE; @@ -49,10 +51,14 @@ public class PrivateDataReminderTest extends PanoramaPublicBaseTest private static final String DELETION_MESSAGE_TITLE = "Title: Data Deletion Requested - "; private static final String MESSAGE_PAGE_TITLE = "Submitted - "; + private Map _originalReminderSettings; @Test public void testPrivateDataReminder() { + // Capture the site wide reminder settings so restoreReminderSettings can put them back. + _originalReminderSettings = getPrivateDataReminderSettings(); + String panoramaPublicProject = PANORAMA_PUBLIC; goToProjectHome(panoramaPublicProject); ApiPermissionsHelper permissionsHelper = new ApiPermissionsHelper(this); @@ -372,6 +378,15 @@ private void postReminders(String projectName, boolean testMode, int expectedExp } + @After + public void restoreReminderSettings() + { + if (_originalReminderSettings != null) + { + restorePrivateDataReminderSettings(_originalReminderSettings, false); + } + } + @Override protected void doCleanup(boolean afterTest) throws TestTimeoutException { diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java index 4aa7a48e..cdc4d8c2 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PublicationSearchTest.java @@ -484,12 +484,7 @@ public void resetAfterTest() if (_originalReminderSettings != null) { - savePrivateDataReminderSettings( - _originalReminderSettings.get("extensionLength"), - _originalReminderSettings.get("delayUntilFirstReminder"), - _originalReminderSettings.get("reminderFrequency"), - Boolean.parseBoolean(_originalReminderSettings.get("enablePublicationSearch")), - null); + restorePrivateDataReminderSettings(_originalReminderSettings, false); } } From 5c8af3ddb0c68fa9657b61960b570f32990e3f96 Mon Sep 17 00:00:00 2001 From: Vagisha Sharma Date: Sun, 4 Oct 2026 21:28:53 -0700 Subject: [PATCH 24/24] Added a stop to the reminder job's publication search when NCBI does not respond * NcbiSearchException reports whether every NCBI request for the search failed. NcbiPublicationSearchServiceImpl sets it when no PMC or PubMed search request completed * PrivateDataReminderJob stops searching for the rest of the run after 3 datasets in a row whose requests all failed, and still posts the reminders. A completed or partly completed search resets the count * The run ends in ERROR when the search was stopped, and the end-of-run summary lists the experiments that were not searched * Added PrivateDataReminderJob.TestCase.testPublicationSearchStopped and NcbiPublicationSearchServiceImpl.TestCase.testSearchReportsWhetherAllRequestsFailed Co-Authored-By: Claude --- .../NcbiPublicationSearchServiceImpl.java | 84 +++++++++++++--- .../ncbi/NcbiSearchException.java | 17 ++++ .../pipeline/PrivateDataReminderJob.java | 97 ++++++++++++++++++- 3 files changed, 178 insertions(+), 20 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java index 5fddc8dd..8342e02f 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiPublicationSearchServiceImpl.java @@ -51,6 +51,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.labkey.panoramapublic.ncbi.PublicationMatch.MATCH_DOI; @@ -216,33 +217,34 @@ public List searchForPublication(@NotNull ExperimentAnnotation maxResults = Math.max(1, Math.min(maxResults, NcbiPublicationSearchService.MAX_RESULTS)); log.info("Starting publication search for experiment: {}", expAnnotations.getId()); - // NCBI requests that failed after retries. An empty result is returned only when every request - // completed, so a caller can distinguish a search that found no publication from one that failed to run. - List failedRequests = new ArrayList<>(); + // An empty result is returned only when every request completed, so a caller can distinguish a search + // that found no publication from one that failed to run. + SearchRequests requests = new SearchRequests(); // Search PubMed Central first - List matchedArticles = searchPmc(expAnnotations, log, failedRequests); + List matchedArticles = searchPmc(expAnnotations, log, requests); // If no PMC results, fall back to PubMed if (matchedArticles.isEmpty()) { log.info("No PMC articles found, trying PubMed fallback"); - matchedArticles = searchPubMed(expAnnotations, log, failedRequests); + matchedArticles = searchPubMed(expAnnotations, log, requests); } - if (!failedRequests.isEmpty()) + if (!requests.failed.isEmpty()) { log.warn("Some NCBI requests did not complete for experiment {}. Failed requests: {}", - expAnnotations.getId(), StringUtils.join(failedRequests, ", ")); + expAnnotations.getId(), StringUtils.join(requests.failed, ", ")); } // Build and return result if (matchedArticles.isEmpty()) { - if (!failedRequests.isEmpty()) + if (!requests.failed.isEmpty()) { throw new NcbiSearchException("Publication search for experiment " + expAnnotations.getId() - + " did not complete. Failed requests: " + StringUtils.join(failedRequests, ", ")); + + " did not complete. Failed requests: " + StringUtils.join(requests.failed, ", "), + !requests.anyCompleted); } log.info("No publications found"); return Collections.emptyList(); @@ -267,11 +269,18 @@ public List searchForPublication(@NotNull ExperimentAnnotation return matchedArticles; } + /** NCBI requests that failed during one search, and whether any request completed. */ + private static class SearchRequests + { + private final List failed = new ArrayList<>(); + private boolean anyCompleted = false; + } + /** * Search PubMed Central */ private @NotNull List searchPmc(@NotNull ExperimentAnnotations expAnnotations, Logger log, - List failedRequests) + SearchRequests requests) { Map searchTermsByStrategy = buildSearchTerms(expAnnotations); @@ -285,6 +294,7 @@ public List searchForPublication(@NotNull ExperimentAnnotation try { List ids = searchPmc(quote(searchTerm), log); + requests.anyCompleted = true; if (!ids.isEmpty()) { pmcIdsByStrategy.put(strategy, ids); @@ -295,7 +305,7 @@ public List searchForPublication(@NotNull ExperimentAnnotation { // Each strategy is a separate request, so the remaining ones are still worth // running. Not logged here, executeSearch logs the query and the cause. - failedRequests.add("PMC search by " + strategy); + requests.failed.add("PMC search by " + strategy); } rateLimit(); } @@ -316,7 +326,7 @@ public List searchForPublication(@NotNull ExperimentAnnotation catch (NcbiSearchException e) { // The IDs cannot be verified without their metadata, so PMC has no usable result. - failedRequests.add("PMC metadata fetch"); + requests.failed.add("PMC metadata fetch"); return Collections.emptyList(); } @@ -619,7 +629,7 @@ private static int countDataIdMatches(PublicationMatch a) * Fall back to PubMed search if PMC finds nothing */ private List searchPubMed(ExperimentAnnotations expAnnotations, Logger log, - List failedRequests) + SearchRequests requests) { String firstName = expAnnotations.getSubmitterUser() != null ? expAnnotations.getSubmitterUser().getFirstName() : null; @@ -645,10 +655,11 @@ private List searchPubMed(ExperimentAnnotations expAnnotations try { pmids = searchPubMed(query, log); + requests.anyCompleted = true; } catch (NcbiSearchException e) { - failedRequests.add("PubMed search"); + requests.failed.add("PubMed search"); return Collections.emptyList(); } @@ -669,7 +680,7 @@ private List searchPubMed(ExperimentAnnotations expAnnotations catch (NcbiSearchException e) { // Metadata is required to confirm that a PMID is a match. - failedRequests.add("PubMed metadata fetch"); + requests.failed.add("PubMed metadata fetch"); return Collections.emptyList(); } @@ -1570,6 +1581,49 @@ protected String executeGet(String url) throws IOException return new NcbiPublicationSearchServiceImpl(client).checkApiKey(apiKey, null); } + @Test + public void testSearchReportsWhetherAllRequestsFailed() + { + // Two PMC search requests, one per term. No submitter, so there is no PubMed fallback. + ExperimentAnnotations expAnnotations = new ExperimentAnnotations(); + expAnnotations.setPxid("PXD000001"); + expAnnotations.setDoi("10.1000/test"); + + NcbiSearchException allFailed = searchFailure(expAnnotations, url -> true); + assertTrue("A search where every request failed should report it", allFailed.isAllRequestsFailed()); + + NcbiSearchException someFailed = searchFailure(expAnnotations, url -> url.contains("PXD000001")); + assertFalse("A search where a request completed should not report that all requests failed", + someFailed.isAllRequestsFailed()); + } + + /** Returns the exception from a search where requests whose URL matches {@code fails} return a 503. */ + private NcbiSearchException searchFailure(ExperimentAnnotations expAnnotations, Predicate fails) + { + NcbiHttpClient client = new NcbiHttpClient.TestCase.NoWaitClient() + { + @Override + protected String executeGet(String url) throws IOException + { + if (fails.test(url)) + { + throw new HttpResponseException(503, "Service Unavailable"); + } + return "{\"esearchresult\":{\"idlist\":[]}}"; + } + }; + try + { + new NcbiPublicationSearchServiceImpl(client).searchForPublication(expAnnotations, LOG); + fail("A search with a failed request and no match should throw NcbiSearchException"); + return null; + } + catch (NcbiSearchException e) + { + return e; + } + } + @Test public void testMockResponses() { diff --git a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java index 4ff46cb9..00d67a86 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java +++ b/panoramapublic/src/org/labkey/panoramapublic/ncbi/NcbiSearchException.java @@ -21,13 +21,30 @@ */ public class NcbiSearchException extends RuntimeException { + private final boolean _allRequestsFailed; + public NcbiSearchException(String message, Throwable cause) { super(message, cause); + _allRequestsFailed = false; } public NcbiSearchException(String message) + { + this(message, false); + } + + public NcbiSearchException(String message, boolean allRequestsFailed) { super(message); + _allRequestsFailed = allRequestsFailed; + } + + /** + * @return true if every NCBI request for the search failed, which suggests NCBI is unavailable. + */ + public boolean isAllRequestsFailed() + { + return _allRequestsFailed; } } diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java index 40c23aba..00642a79 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/PrivateDataReminderJob.java @@ -69,6 +69,8 @@ public class PrivateDataReminderJob extends PipelineJob { private static final int MIN_DATASETS_FOR_FAILURE = 3; private static final double PUBLICATION_SEARCH_FAILURE_THRESHOLD = 0.5; + // Consecutive datasets with all NCBI requests failed before the search is stopped for the run. + private static final int MAX_CONSECUTIVE_ALL_FAILED = 3; /** * @return true when the publication search failure rate reaches the threshold, which suggests a @@ -247,12 +249,21 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp return null; } + if (results.isPublicationSearchStopped()) + { + // The search deferral is not reset, so the next run searches again. + log.info("Publication search is stopped for this run. Not re-searching for experiment {}", expAnnotations.getId()); + results.addPublicationSearchSkipped(expAnnotations.getId()); + return null; + } + // Search deferral expired — re-search NCBI log.info("Search deferral expired for experiment {} (dismissed {}); re-searching NCBI", expAnnotations.getId(), dismissedDate); try { results.addPublicationSearchAttempted(); PublicationMatch newMatch = NcbiPublicationSearchService.get().searchForPublication(expAnnotations, log); + results.addPublicationSearchCompleted(); if (newMatch != null && !newMatch.getPublicationId().equals(datasetStatus.getPotentialPublicationId())) { // Different publication found — return it (caller will save and notify) @@ -296,12 +307,21 @@ private PublicationMatch searchForPublication(@NotNull ExperimentAnnotations exp } } + if (results.isPublicationSearchStopped()) + { + log.info("Publication search is stopped for this run. Not searching for experiment {}", expAnnotations.getId()); + results.addPublicationSearchSkipped(expAnnotations.getId()); + return null; + } + // Perform the publication search log.info("Searching for publications for experiment {}", expAnnotations.getId()); try { results.addPublicationSearchAttempted(); - return NcbiPublicationSearchService.get().searchForPublication(expAnnotations, log); + PublicationMatch match = NcbiPublicationSearchService.get().searchForPublication(expAnnotations, log); + results.addPublicationSearchCompleted(); + return match; } catch (NcbiSearchException e) { @@ -368,8 +388,8 @@ private boolean ncbiApiKeyAccepted() /** * @return error when the job could not start, a dataset that should have been sent a reminder was - * not, or the publication search failed for half or more of the datasets it ran for. Cancelled when - * the job was cancelled with nothing else to report, and complete otherwise. + * not, or the publication search failed for half or more of the datasets it ran for or was stopped. Cancelled + * when the job was cancelled with nothing else to report, and complete otherwise. */ private TaskStatus postMessage(List expAnnotationIds, Journal panoramaPublic) { @@ -397,7 +417,8 @@ private TaskStatus postMessage(List expAnnotationIds, Journal panoramaP // An ERROR logged through the job's logger sets the status to error, and the status set here // would overwrite it. Report the errors the job recorded instead. - if (processingResults.getTotalErrors() > 0 || processingResults.publicationSearchFailingWidely()) + if (processingResults.getTotalErrors() > 0 || processingResults.publicationSearchFailingWidely() + || processingResults.isPublicationSearchStopped()) { return TaskStatus.error; } @@ -782,6 +803,9 @@ private static class ProcessingResults private final List _submitterNotFound = new ArrayList<>(); private final List _publicationSearchFailed = new ArrayList<>(); private int _publicationSearchAttempted = 0; + private int _consecutiveAllFailed = 0; + private boolean _publicationSearchStopped = false; + private final List _publicationSearchSkipped = new ArrayList<>(); private final List _processingFailed = new ArrayList<>(); private final List _skipped = new ArrayList<>(); private int _processed = 0; @@ -835,10 +859,34 @@ public void addPublicationSearchAttempted() _publicationSearchAttempted++; } - public void addPublicationSearchFailed(Integer experimentId, Exception e) + public void addPublicationSearchCompleted() + { + _consecutiveAllFailed = 0; + } + + public void addPublicationSearchFailed(Integer experimentId, NcbiSearchException e) { _publicationSearchFailed.add(experimentId); _log.warn("Publication search failed for experiment Id: {}. A reminder was still posted. {}", experimentId, e.getMessage(), e); + + // Some requests completing means NCBI is responding, so the failure is intermittent. + _consecutiveAllFailed = e.isAllRequestsFailed() ? _consecutiveAllFailed + 1 : 0; + if (!_publicationSearchStopped && _consecutiveAllFailed >= MAX_CONSECUTIVE_ALL_FAILED) + { + _publicationSearchStopped = true; + _log.error("Every NCBI request failed for {} datasets in a row. The publication search is stopped for the rest of this run. Reminders are still posted.", + _consecutiveAllFailed); + } + } + + public boolean isPublicationSearchStopped() + { + return _publicationSearchStopped; + } + + public void addPublicationSearchSkipped(Integer experimentId) + { + _publicationSearchSkipped.add(experimentId); } public void addProcessingFailed(Integer experimentId, Exception e) @@ -903,6 +951,13 @@ public void logSkipped(Logger log) } } + if (_publicationSearchStopped) + { + log.error("The publication search was stopped after every NCBI request failed for {} datasets in a row. Reminders were still posted. Experiment Ids not searched: {}", + MAX_CONSECUTIVE_ALL_FAILED, + _publicationSearchSkipped.isEmpty() ? "none" : StringUtils.join(_publicationSearchSkipped, ", ")); + } + if (!_processingFailed.isEmpty()) { log.error("Processing failed for the following experiment Ids: {}", StringUtils.join(_processingFailed, ", ")); @@ -1000,6 +1055,38 @@ public void testGetTotalErrors() assertEquals("A failed publication search is not a missed reminder", 5, results.getTotalErrors()); } + @Test + public void testPublicationSearchStopped() + { + ProcessingResults results = new ProcessingResults(10, TEST_LOG); + + // Two datasets in a row with every request failed, then a search that completed. + results.addPublicationSearchFailed(1, allRequestsFailed()); + results.addPublicationSearchFailed(2, allRequestsFailed()); + results.addPublicationSearchCompleted(); + results.addPublicationSearchFailed(3, allRequestsFailed()); + assertFalse("A completed search should reset the count", results.isPublicationSearchStopped()); + + // A partial failure resets the count. + results.addPublicationSearchFailed(4, allRequestsFailed()); + results.addPublicationSearchFailed(5, new NcbiSearchException("test", false)); + results.addPublicationSearchFailed(6, allRequestsFailed()); + assertFalse("A search where some requests completed should reset the count", results.isPublicationSearchStopped()); + + results.addPublicationSearchFailed(7, allRequestsFailed()); + assertFalse("Two in a row should not stop the search", results.isPublicationSearchStopped()); + results.addPublicationSearchFailed(8, allRequestsFailed()); + assertTrue("Three in a row with every request failed should stop the search", results.isPublicationSearchStopped()); + + // Stopping the search sets the status, not the error count. + assertEquals("Stopping the search should not count as a missed reminder", 0, results.getTotalErrors()); + } + + private static NcbiSearchException allRequestsFailed() + { + return new NcbiSearchException("test", true); + } + @Test public void testAnnouncementNotFoundSplitsOnAnnouncementId() {