Search NCBI for papers associated with private datasets (#606)

- Added a setting in the admin console (Private Data Reminder Settings) to enable publication search
- Added NcbiPublicationSearchService that searches for publications in
  -- PubMed Central (full-text available): search terms ProteomeXchange Id, Panorama Public short URL, Panorama Public DOI.  Author and title verified.
  -- PubMed - fallback if no matches found in PMC; search author and title only since PubMed does not have full-text articles
  -- Preprints (e.g. biorxiv, medrxiv etc.) are filtered out
  -- Results are prioritized by number of matched search terms (e.g. PX ID, Panorama link) as well as paper publication date proximity to data submission date
- Publication search happens automatically in the PrivateDataReminderJob  when enabled in the admin console
- Publications for a single dataset can also be searched through the Search Publications menu item in the _TargetedMS Experiment_ webpart menu
- If a publication match is found during the PrivateDataReminderJob,  instead of the usual reminder message, a "publication found" message is posted to the submitter
- Submitter can dismiss suggested publication by clicking the Dismiss Publication Suggestion link in the message. 
- DatasetStatus caches publication result to avoid repeated calls to NCBI's EUtils endpoints
- Moved NCBI citation lookup methods to NcbiPublicationSearchServiceImpl
- **Tests**
   -- Unit tests added in NcbiPublicationSearchServiceImpl
   -- Added Selenium test PublicationSearchTest
   -- Test uses a mock NCBI service on TeamCity (mocking only the outbound HTTP requests) so all real search and filtering logic is exercised without live API calls.

Author: vagisha

panoramapublic/resources/schemas/dbscripts/postgresql/panoramapublic-25.003-25.004.sql

@@ -0,0 +1,7 @@
+
+-- Add publication tracking columns to DatasetStatus
+ALTER TABLE panoramapublic.DatasetStatus ADD COLUMN PotentialPublicationId VARCHAR(255);
+ALTER TABLE panoramapublic.DatasetStatus ADD COLUMN PublicationType VARCHAR(50);
+ALTER TABLE panoramapublic.DatasetStatus ADD COLUMN PublicationMatchInfo TEXT;
+ALTER TABLE panoramapublic.DatasetStatus ADD COLUMN Citation TEXT;
+ALTER TABLE panoramapublic.DatasetStatus ADD COLUMN UserDismissedPublication TIMESTAMP;

panoramapublic/resources/schemas/panoramapublic.xml

@@ -864,6 +864,21 @@
             <column columnName="LastReminderDate" />
             <column columnName="ExtensionRequestedDate" />
             <column columnName="DeletionRequestedDate"/>
+            <column columnName="PotentialPublicationId">
+                <description>Publication ID (PubMed ID, PMC ID) suggestion based on NCBI search</description>
+            </column>
+            <column columnName="PublicationType">
+                <description>Type of publication ID: PMID, PMC</description>
+            </column>
+            <column columnName="PublicationMatchInfo">
+                <description>Information about which fields (e.g. PXD, Panorama link, title etc.) matched</description>
+            </column>
+            <column columnName="Citation">
+                <description>NLM citation string for the potential publication match</description>
+            </column>
+            <column columnName="UserDismissedPublication">
+                <description>Date when the user dismissed the publication suggestion for this dataset</description>
+            </column>
         </columns>
     </table>
 </tables>

panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java

@@ -47,6 +47,7 @@
 import org.labkey.panoramapublic.bluesky.PanoramaPublicLogoResourceType;
 import org.labkey.panoramapublic.catalog.CatalogImageAttachmentType;
 import org.labkey.panoramapublic.message.PrivateDataReminderSettings;
+import org.labkey.panoramapublic.ncbi.NcbiPublicationSearchServiceImpl;
 import org.labkey.panoramapublic.model.Journal;
 import org.labkey.panoramapublic.model.speclib.SpecLibKey;
 import org.labkey.panoramapublic.pipeline.CopyExperimentPipelineProvider;
@@ -92,7 +93,7 @@ public String getName()
     @Override
     public @Nullable Double getSchemaVersion()
     {
-        return 25.003;
+        return 25.004;
     }
 
     @Override
@@ -382,6 +383,7 @@ public Set<String> getSchemaNames()
         set.add(CatalogEntryManager.TestCase.class);
         set.add(BlueskyApiClient.TestCase.class);
         set.add(PrivateDataReminderSettings.TestCase.class);
+        set.add(NcbiPublicationSearchServiceImpl.TestCase.class);
 
         return set;
     }

panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java

@@ -28,6 +28,7 @@
 import org.labkey.panoramapublic.model.JournalExperiment;
 import org.labkey.panoramapublic.model.JournalSubmission;
 import org.labkey.panoramapublic.model.Submission;
+import org.labkey.panoramapublic.ncbi.PublicationMatch;
 import org.labkey.panoramapublic.proteomexchange.ProteomeXchangeService;
 import org.labkey.panoramapublic.query.ExperimentAnnotationsManager;
 import org.labkey.panoramapublic.query.JournalManager;
@@ -358,22 +359,63 @@ public static void postDataDeletionRequestMessage(@NotNull Journal journal, @Not
         postNotificationFullTitle(journal, je, messageBody.toString(), journalAdmin, messageTitle, AnnouncementService.StatusOption.Active, null);
     }
 
+    public static void postPublicationDismissalMessage(@NotNull Journal journal, @NotNull JournalExperiment je, @NotNull ExperimentAnnotations expAnnotations, User submitter,
+                                                       @NotNull PublicationMatch publicationMatch)
+    {
+        User journalAdmin = JournalManager.getJournalAdminUser(journal);
+        if (journalAdmin == null)
+        {
+            throw new NotFoundException(String.format("Could not find an admin user for %s.", journal.getName()));
+        }
+
+        String messageTitle = "Publication Suggestion Dismissed" +" - " + expAnnotations.getShortUrl().renderShortURL();
+
+        StringBuilder messageBody = new StringBuilder();
+        messageBody.append("Dear ").append(getUserName(submitter)).append(",").append(NL2);
+        messageBody.append("Thank you for letting us know that the suggested paper is not associated with your data on Panorama Public.");
+
+        messageBody.append(NL2).append(bold("Dismissed Publication:")).append(" ").append(publicationMatch.getPublicationIdLabel());
+        if (!StringUtils.isBlank(publicationMatch.getCitation()))
+        {
+            messageBody.append(NL).append(publicationMatch.getCitation());
+        }
+
+        messageBody.append(NL2).append("We will no longer suggest this paper for your dataset. ")
+                .append("If you would like to make your data public, you can do so at any time ")
+                .append("by clicking the \"Make Public\" button in your data folder, or by clicking this link: ")
+                .append(bold(link("Make Data Public", PanoramaPublicController.getMakePublicUrl(expAnnotations.getId(), expAnnotations.getContainer()).getURIString())))
+                .append(".");
+        messageBody.append(NL2).append("Best regards,");
+        messageBody.append(NL).append(getUserName(journalAdmin));
+
+        postNotificationFullTitle(journal, je, messageBody.toString(), journalAdmin, messageTitle, AnnouncementService.StatusOption.Closed, null);
+    }
+
 
     public static void postPrivateDataReminderMessage(@NotNull Journal journal, @NotNull JournalSubmission js, @NotNull ExperimentAnnotations expAnnotations,
                                                       @NotNull User submitter, @NotNull User messagePoster, List<User> notifyUsers,
-                                                      @NotNull Announcement announcement, @NotNull Container announcementsContainer, @NotNull User journalAdmin)
+                                                      @NotNull Announcement announcement, @NotNull Container announcementsContainer, @NotNull User journalAdmin,
+                                                      @Nullable PublicationMatch articleMatch)
     {
-        String message = getDataStatusReminderMessage(expAnnotations, submitter, js, announcement, announcementsContainer, journalAdmin);
-        String title = "Action Required: Status Update for Your Private Data on Panorama Public";
+        String message = getDataStatusReminderMessage(expAnnotations, submitter, js, announcement, announcementsContainer, journalAdmin, articleMatch);
+        String title = articleMatch != null
+                ? "Action Required: Publication Found for Your Data on Panorama Public"
+                : "Action Required: Status Update for Your Private Data on Panorama Public";
         postNotificationFullTitle(journal, js.getJournalExperiment(), message, messagePoster, title, AnnouncementService.StatusOption.Closed, notifyUsers);
     }
 
     public static String getDataStatusReminderMessage(@NotNull ExperimentAnnotations exptAnnotations, @NotNull User submitter,
                                                       @NotNull JournalSubmission js,@NotNull Announcement announcement,
-                                                      @NotNull Container announcementContainer, @NotNull User journalAdmin)
+                                                      @NotNull Container announcementContainer, @NotNull User journalAdmin,
+                                                      @Nullable PublicationMatch articleMatch)
     {
         String shortUrl = exptAnnotations.getShortUrl().renderShortURL();
-        String makePublicLink = PanoramaPublicController.getMakePublicUrl(exptAnnotations.getId(), exptAnnotations.getContainer()).getURIString();
+        ActionURL makePublicUrl = PanoramaPublicController.getMakePublicUrl(exptAnnotations.getId(), exptAnnotations.getContainer());
+        if (articleMatch != null && articleMatch.isPubMed() && articleMatch.getPublicationId() != null)
+        {
+            makePublicUrl.replaceParameter("pubmedId", articleMatch.getPublicationId());
+        }
+        String makePublicLink = makePublicUrl.getURIString();
         String dateString = DateUtil.formatDateTime(js.getLatestSubmission().getCreated(), PrivateDataReminderSettings.DATE_FORMAT_PATTERN);
 
         ActionURL viewMessageUrl = new ActionURL("announcements", "thread", announcementContainer)
@@ -386,32 +428,55 @@ public static String getDataStatusReminderMessage(@NotNull ExperimentAnnotations
         ActionURL requestExtensionUrl = new ActionURL(PanoramaPublicController.RequestExtensionAction.class, exptAnnotations.getContainer())
                 .addParameter("shortUrlEntityId", shortUrlEntityId);
 
-        ActionURL requesDeletionUrl = new ActionURL(PanoramaPublicController.RequestDeletionAction.class, exptAnnotations.getContainer())
+        ActionURL requestDeletionUrl = new ActionURL(PanoramaPublicController.RequestDeletionAction.class, exptAnnotations.getContainer())
                 .addParameter("shortUrlEntityId",shortUrlEntityId);
 
+        ActionURL dismissPublicationUrl = new ActionURL(PanoramaPublicController.DismissPublicationSuggestionAction.class, exptAnnotations.getContainer())
+                .addParameter("shortUrlEntityId", shortUrlEntityId);
 
         ExperimentAnnotations sourceExperiment = ExperimentAnnotationsManager.get(exptAnnotations.getSourceExperimentId());
 
         StringBuilder message = new StringBuilder();
-        message.append("Dear ").append(getUserName(submitter)).append(",").append(NL2)
-                .append("We are reaching out regarding your data on Panorama Public (").append(shortUrl).append("), which has been private since ")
-                .append(dateString).append(".")
-                .append(NL2).append(bold("Title:")).append(" ").append(escape(exptAnnotations.getTitle()))
-                .append(NL2).append(bold("Is the paper associated with this work already published?"))
-                .append(NL).append("- If yes: Please make your data public by clicking the \"Make Public\" button in your folder or by clicking this link: ")
-                .append(bold(link("Make Data Public", makePublicLink)))
-                .append(". This helps ensure that your valuable research is easily accessible to the community.")
-                .append(NL).append("- If not: You have a couple of options:")
-                .append(NL).append("  - ").append(bold("Request an Extension")).append(" - If your paper is still under review, or you need additional time, please let us know by clicking ")
-                .append(bold(link("Request Extension", requestExtensionUrl.getURIString()))).append(".")
-                .append(NL).append("  - ").append(bold("Delete from Panorama Public")).append(" - If you no longer wish to host your data on Panorama Public, please click ")
-                .append(bold(link("Request Deletion", requesDeletionUrl.getURIString()))).append(". ")
-                .append("We will remove your data from Panorama Public.");
-        if (sourceExperiment != null)
+        message.append("Dear ").append(getUserName(submitter)).append(",").append(NL2);
+
+        if (articleMatch != null)
+        {
+            // Message variant when a publication was found
+            message.append("We found a paper that appears to be associated with your private data on Panorama Public (").append(shortUrl).append(").")
+                    .append(NL2).append(bold("Title:")).append(" ").append(escape(exptAnnotations.getTitle()))
+                    .append(NL2).append(bold("Publication Found:")).append(" ")
+                    .append(articleMatch.getCitation() != null
+                            ? link(articleMatch.getCitation(), articleMatch.getPublicationUrl())
+                            : link(articleMatch.getPublicationIdLabel(), articleMatch.getPublicationUrl()))
+                    .append(NL2).append("If this is indeed your paper, congratulations! We encourage you to make your data public so the research community can access it alongside your paper. ")
+                    .append("You can do this by clicking the \"Make Public\" button in your data folder or by clicking this link: ")
+                    .append(bold(link("Make Data Public", makePublicLink))).append(".")
+                    .append(articleMatch.isPubMed() ? " Please enter " + articleMatch.getPublicationId() + " in the PubMed ID field." : "")
+                    .append(NL2).append("If this paper is not associated with your data please let us know by clicking ")
+                    .append(bold(link("Dismiss Publication Suggestion", dismissPublicationUrl.getURIString())));
+        }
+        else
         {
-            message.append(" However, your source folder (")
-                    .append(getContainerLink(sourceExperiment.getContainer()))
-                    .append(") will remain intact, allowing you to resubmit your data in the future if you wish.");
+            // Original message variant when no publication was found
+            message.append("We are reaching out regarding your data on Panorama Public (").append(shortUrl).append("), which has been private since ")
+                    .append(dateString).append(".")
+                    .append(NL2).append(bold("Title:")).append(" ").append(escape(exptAnnotations.getTitle()))
+                    .append(NL2).append(bold("Is the paper associated with this work already published?"))
+                    .append(NL).append("- If yes: Please make your data public by clicking the \"Make Public\" button in your folder or by clicking this link: ")
+                    .append(bold(link("Make Data Public", makePublicLink)))
+                    .append(". This helps ensure that your valuable research is easily accessible to the community.")
+                    .append(NL).append("- If not: You have a couple of options:")
+                    .append(NL).append("  - ").append(bold("Request an Extension")).append(" - If your paper is still under review, or you need additional time, please let us know by clicking ")
+                    .append(bold(link("Request Extension", requestExtensionUrl.getURIString()))).append(".")
+                    .append(NL).append("  - ").append(bold("Delete from Panorama Public")).append(" - If you no longer wish to host your data on Panorama Public, please click ")
+                    .append(bold(link("Request Deletion", requestDeletionUrl.getURIString()))).append(". ")
+                    .append("We will remove your data from Panorama Public.");
+            if (sourceExperiment != null)
+            {
+                message.append(" However, your source folder (")
+                        .append(getContainerLink(sourceExperiment.getContainer()))
+                        .append(") will remain intact, allowing you to resubmit your data in the future if you wish.");
+            }
         }
 
         message.append(NL2).append("If you have any questions or need further assistance, please do not hesitate to respond to this message by ")