Merge 26.3 to develop

Author: labkey-teamcity

api/src/org/labkey/api/reports/report/ScriptEngineReport.java

@@ -60,8 +60,7 @@
 import org.labkey.api.reports.report.r.view.TsvOutput;
 import org.labkey.api.thumbnail.Thumbnail;
 import org.labkey.api.util.FileUtil;
-import org.labkey.api.util.Path;
-import org.labkey.api.util.UnexpectedException;
+import org.labkey.api.util.UnexpectedException;
 import org.labkey.api.view.HttpView;
 import org.labkey.api.view.VBox;
 import org.labkey.api.view.ViewContext;
@@ -71,8 +70,7 @@
 
 import javax.script.ScriptEngine;
 import javax.script.ScriptException;
-import java.io.File;
-import java.io.IOException;
+import java.io.IOException;
 import java.io.PrintWriter;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
@@ -212,36 +210,41 @@ public FileLike createInputDataFile(@NotNull ViewContext context) throws SQLExce
      * Note: This method used to stash results in members (_tempFolder and _tempFolderPipeline), but that no longer works
      * now that we cache reports between threads (e.g., Thread.currentThread().getId() is part of the path).
      */
-    public FileLike getReportDirFileLike(@NotNull String executingContainerId)
-    {
-        boolean isPipeline = BooleanUtils.toBoolean(getDescriptor().getProperty(ScriptReportDescriptor.Prop.runInBackground));
-        return getReportDirFileLike(executingContainerId, isPipeline);
-    }
-
-    protected FileLike getReportDirFileLike(@NotNull String executingContainerId, boolean isPipeline)
-    {
-        FileLike tempRoot = getTempRootFileLike(getDescriptor());
-        String reportId = FileUtil.makeLegalName(String.valueOf(getDescriptor().getReportId())).replaceAll(" ", "_");
-        FileLike tempFolder;
-
-        try
-        {
-            if (isPipeline)
-            {
-                String identifier = RReportJob.getJobIdentifier();
-                if (identifier != null)
-                    tempFolder = FileUtil.appendPath(tempRoot,
-                            Path.parse(executingContainerId + File.separator + "Report_" + reportId + File.separator + identifier));
-                else
-                    tempFolder = FileUtil.appendPath(tempRoot,
-                            Path.parse(executingContainerId + File.separator + "Report_" + reportId));
-            }
-            else
-                tempFolder = FileUtil.appendPath(tempRoot,
-                        Path.parse(executingContainerId + File.separator + "Report_" + reportId + File.separator + Thread.currentThread().getId()));
-
-            FileUtil.mkdirs(tempFolder);
-            return tempFolder;
+    public FileLike getReportDirFileLike(@NotNull String executingContainerId)
+    {
+        boolean isPipeline = BooleanUtils.toBoolean(getDescriptor().getProperty(ScriptReportDescriptor.Prop.runInBackground));
+        return getReportDirFileLike(executingContainerId, isPipeline);
+    }
+
+    protected FileLike getBackgroundOutputDirFileLike(@NotNull String executingContainerId)
+    {
+        FileLike tempRoot = getTempRootFileLike(getDescriptor());
+        String reportId = FileUtil.makeLegalName(String.valueOf(getDescriptor().getReportId())).replaceAll(" ", "_");
+
+        // Build FileLike paths from VFS path segments instead of File.separator so Windows
+        // backslashes do not get folded into a single org.labkey.api.util.Path segment.
+        return tempRoot.resolveChild(executingContainerId).resolveChild("Report_" + reportId);
+    }
+
+    protected FileLike getReportDirFileLike(@NotNull String executingContainerId, boolean isPipeline)
+    {
+        FileLike tempFolder;
+
+        try
+        {
+            tempFolder = getBackgroundOutputDirFileLike(executingContainerId);
+
+            if (isPipeline)
+            {
+                String identifier = RReportJob.getJobIdentifier();
+                if (identifier != null)
+                    tempFolder = tempFolder.resolveChild(identifier);
+            }
+            else
+                tempFolder = tempFolder.resolveChild(String.valueOf(Thread.currentThread().getId()));
+
+            FileUtil.mkdirs(tempFolder);
+            return tempFolder;
         }
         catch (IOException e)
         {

api/src/org/labkey/api/reports/report/r/RReport.java

@@ -696,6 +696,20 @@ public FileLike getReportDirFileLike(@NotNull String executingContainerId)
         return super.getReportDirFileLike(executingContainerId);
     }
 
+    @Override
+    protected FileLike getBackgroundOutputDirFileLike(@NotNull String executingContainerId)
+    {
+        FileLike reportDir = null;
+
+        if (getKnitrFormat() != RReportDescriptor.KnitrFormat.None)
+            reportDir = getCacheDir(executingContainerId);
+
+        if (reportDir != null)
+            return reportDir;
+
+        return super.getBackgroundOutputDirFileLike(executingContainerId);
+    }
+
     @Override
     public void clearCache()
     {

api/src/org/labkey/api/reports/report/r/RReportJob.java

@@ -42,7 +42,6 @@
 
 import java.io.File;
 import java.io.Serializable;
-import java.nio.file.CopyOption;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -275,16 +274,22 @@ protected void processOutputs(RReport report, List<ParamReplacement> outputSubst
             {
                 // write the output substitution map to disk so we can render the view later
                 FileLike reportDir = report.getReportDirFileLike(getJob().getContainerId());
+                FileLike parentDir = report.getBackgroundOutputDirFileLike(getJob().getContainerId());
                 FileLike substitutionMap;
 
-                if (reportDir.getName().equals(getJobIdentifier()))
+                // ScriptEngineReport#getReportDirFileLike() appends the ThreadLocal job identifier only
+                // when background execution needs a per-job child directory under parentDir.
+                boolean hasJobSpecificReportDir = !reportDir.equals(parentDir);
+                if (hasJobSpecificReportDir)
                 {
-                    FileLike parentDir = reportDir.getParent();
                     // clean up the destination folder
                     for (FileLike file : parentDir.getChildren())
                     {
-                        if (!file.isDirectory() && !"log".equalsIgnoreCase(FileUtil.getExtension(file)))
+                        if (file.isFile() && !"log".equalsIgnoreCase(FileUtil.getExtension(file)))
+                        {
+                            getJob().debug("deleting parent file=" + file);
                             file.delete();
+                        }
                     }
 
                     // rewrite the parameter replacement files to point to the destination folder
@@ -294,6 +299,7 @@ protected void processOutputs(RReport report, List<ParamReplacement> outputSubst
                         for (FileLike file : replacement.getFiles())
                         {
                             FileLike newFile = parentDir.resolveChild(file.getName());
+                            getJob().debug("move replacement file  =" + replacement.getName() + " from=" + file + " to=" + newFile);
                             file.move(newFile);
                             newFiles.add(newFile);
                         }
@@ -317,6 +323,7 @@ protected void processOutputs(RReport report, List<ParamReplacement> outputSubst
                             {
                                 newFile = FileUtil.createTempFile(LOG_FILE_PREFIX, ".log", parentDir);
                                 getJob().setLogFile(newFile);
+                                getJob().debug("copy log file from=" + file + " to=" + newFile);
                                 FileUtil.copyFile(file, newFile, StandardCopyOption.REPLACE_EXISTING);
                             }
                             // report.log != getLogFile(), just regular file
@@ -325,16 +332,19 @@ protected void processOutputs(RReport report, List<ParamReplacement> outputSubst
                                 String logFileContent = PageFlowUtil.getStreamContentsAsString(file.openInputStream());
                                 if (!StringUtils.isEmpty(logFileContent))
                                     getJob().info("REPORT.LOG CONTENTS:\n" + logFileContent);
+                                getJob().debug("move log file from=" + file + " to=" + newFile);
                                 file.move(newFile);
                             }
                         }
                         else
                         {
+                            getJob().debug("move file from=" + file + " to=" + newFile);
                             file.move(newFile);
                         }
                     }
+                    getJob().debug("delete reportDir=" + reportDir);
                     FileUtil.deleteDir(reportDir);
-                    substitutionMap = reportDir.getParent().resolveChild(RReport.SUBSTITUTION_MAP);
+                    substitutionMap = parentDir.resolveChild(RReport.SUBSTITUTION_MAP);
                 }
                 else
                     substitutionMap = reportDir.resolveChild(RReport.SUBSTITUTION_MAP);

core/src/org/labkey/core/query/DocumentsTable.java

@@ -2,6 +2,7 @@
 
 import org.jetbrains.annotations.NotNull;
 import org.labkey.api.attachments.AttachmentService;
+import org.labkey.api.collections.CsvSet;
 import org.labkey.api.collections.LabKeyCollectors;
 import org.labkey.api.data.BaseColumnInfo;
 import org.labkey.api.data.ContainerFilter;
@@ -11,6 +12,7 @@
 import org.labkey.api.data.SQLFragment;
 import org.labkey.api.data.dialect.SqlDialect;
 import org.labkey.api.query.AliasManager;
+import org.labkey.api.query.FieldKey;
 import org.labkey.api.query.FilteredTable;
 import org.labkey.api.query.UserIdQueryForeignKey;
 
@@ -26,16 +28,21 @@ public DocumentsTable(@NotNull CoreQuerySchema userSchema, ContainerFilter cf)
         getMutableColumnOrThrow("ModifiedBy").setHidden(true);
         getMutableColumnOrThrow("Modified").setHidden(true);
         MutableColumnInfo owner = getMutableColumnOrThrow("Owner");
-        owner.setHidden(true);
         owner.setFk(new UserIdQueryForeignKey(_userSchema));
         getMutableColumnOrThrow("Parent").setHidden(true);
         getMutableColumnOrThrow("DocumentSize").setFormat("#,##0");
         getMutableColumnOrThrow("Document").setHidden(true);
         getMutableColumnOrThrow("LastIndexed").setHidden(true);
         addColumn(new BaseColumnInfo("ParentDescription", this, JdbcType.VARCHAR));
         BaseColumnInfo orphaned = new BaseColumnInfo("Orphaned", this, JdbcType.BOOLEAN);
-        orphaned.setHidden(true);
         addColumn(orphaned);
+
+        setDefaultVisibleColumns(
+            new CsvSet("CreatedBy, Created, Container, DocumentName, DocumentSize, DocumentType, ParentType, ParentDescription")
+                .stream()
+                .map(FieldKey::fromParts)
+                .toList()
+        );
     }
 
     @Override

core/src/org/labkey/core/view/template/bootstrap/header.jsp

@@ -143,12 +143,12 @@
     {
 %>
             <li class="navbar-search hidden-xs">
-                <a class="fa fa-search" id="global-search-trigger" aria-label="<%=h(SearchUtils.getPlaceholder(c))%>"></a>
+                <a class="fa fa-search" id="global-search-trigger" aria-label="<%=h(SearchUtils.getPlaceholder(c))%>" role="button"></a>
                 <div id="global-search" class="global-search">
                     <labkey:form id="global-search-form" action="<%=urlProvider(SearchUrls.class).getSearchURL(c, null)%>" method="GET">
                         <input type="text" class="search-box" name="q" placeholder="<%=h(SearchUtils.getPlaceholder(c))%>" value="">
                         <input type="submit" hidden>
-                        <a id="a_header_search" href="#" class="btn-search fa fa-search"></a>
+                        <a id="a_header_search" href="#" class="btn-search fa fa-search" aria-label="Search" role="button"></a>
                     </labkey:form>
                     <% pageConfig.addHandler("a_header_search","click","document.getElementById('global-search-form').submit(); return false;"); %>
                 </div>

core/webapp/internal/tippy.lib.xml

@@ -1,6 +1,7 @@
 <libraries xmlns="http://labkey.org/clientLibrary/xml/">
     <library compileInProductionMode="false">
+        <script path="internal/tippy/popper.min.js"/>
         <script path="internal/tippy/tippy-bundle.umd.js" mode="dev"/>
-        <script path="internal/tippy/popper.min.js" mode="dev"/>
+        <script path="internal/tippy/tippy-bundle.umd.min.js" mode="production"/>
     </library>
 </libraries>

core/webapp/internal/tippy/tippy-bundle.umd.min.js

@@ -576,10 +576,10 @@ public void delete(User user, @Nullable String auditUserComment) throws DomainNo
         try (DbScope.Transaction transaction = (table != null) ? table.getSchema().getScope().ensureTransaction() :
              ExperimentService.get().ensureTransaction())
         {
-            // remove related attachments, discussions, and indices
+            // remove related full-text search docs and attachments
             ListManager.get().deleteIndexedList(this);
             if (qus instanceof ListQueryUpdateService listQus)
-                listQus.deleteRelatedListData(user, getContainer());
+                listQus.deleteRelatedListData(null);
 
             // then delete the list itself
             ListManager.get().deleteListDef(getContainer(), getListId());

list/src/org/labkey/list/model/ListDefinitionImpl.java

@@ -26,6 +26,7 @@
 import org.labkey.api.audit.AuditLogService;
 import org.labkey.api.audit.TransactionAuditProvider;
 import org.labkey.api.collections.CaseInsensitiveHashMap;
+import org.labkey.api.collections.CsvSet;
 import org.labkey.api.data.ColumnInfo;
 import org.labkey.api.data.CompareType;
 import org.labkey.api.data.Container;
@@ -90,6 +91,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.labkey.api.util.IntegerUtils.isIntegral;
 
@@ -759,45 +761,39 @@ protected Map<String, Object> deleteRow(User user, Container container, Map<Stri
         return result;
     }
 
+    record ListRow(Container container, String entityId){}
 
-    // Deletes attachments & discussions, and removes list documents from full-text search index.
-    public void deleteRelatedListData(final User user, final Container container)
+    // Delete attachments and remove documents from the full-text search index. If Container is non-null, delete only
+    // from that container (truncate case). If container is null, delete from all containers (delete list case).
+    public void deleteRelatedListData(final @Nullable Container container)
     {
         // Unindex all item docs and the entire list doc
         ListManager.get().deleteIndexedList(_list);
 
-        // Delete attachments and discussions associated with a list in batches of 1,000
-        new TableSelector(getDbTable(), Collections.singleton("entityId"), SimpleFilter.createContainerFilter(container), null).forEachBatch(String.class, 1000, new ForEachBatchBlock<>()
+        if (hasAttachmentProperties())
         {
-            @Override
-            public boolean accept(String entityId)
-            {
-                return null != entityId;
-            }
+            SimpleFilter filter = null != container ? SimpleFilter.createContainerFilter(container) : null;
 
-            @Override
-            public void exec(List<String> entityIds)
+            // Delete attachments associated with a list in batches of 1,000
+            new TableSelector(getDbTable(), new CsvSet("Container, EntityId"), filter, null).forEachBatch(ListRow.class, 1000, new ForEachBatchBlock<>()
             {
-                // delete the related list data for this block
-                deleteRelatedListData(user, container, entityIds);
-            }
-        });
-    }
-
-    // delete the related list data for this block of entityIds
-    private void deleteRelatedListData(User user, Container container, List<String> entityIds)
-    {
-        // Build up set of entityIds and AttachmentParents
-        List<AttachmentParent> attachmentParents = new ArrayList<>();
+                @Override
+                public boolean accept(ListRow row)
+                {
+                    return null != row.entityId();
+                }
 
-        // Delete Attachments
-        if (hasAttachmentProperties())
-        {
-            for (String entityId : entityIds)
-            {
-                attachmentParents.add(new ListItemAttachmentParent(entityId, container));
-            }
-            AttachmentService.get().deleteAttachments(attachmentParents);
+                @Override
+                public void exec(List<ListRow> rows)
+                {
+                    // delete the related attachments for this block
+                    AttachmentService.get().deleteAttachments(
+                        rows.stream()
+                            .map(row -> new ListItemAttachmentParent(row.entityId(), row.container()))
+                            .collect(Collectors.toList())
+                    );
+                }
+            });
         }
     }
 
@@ -807,7 +803,7 @@ protected int truncateRows(User user, Container container) throws QueryUpdateSer
         int result;
         try (DbScope.Transaction transaction = getDbTable().getSchema().getScope().ensureTransaction())
         {
-            deleteRelatedListData(user, container);
+            deleteRelatedListData(container);
             result = super.truncateRows(getListUser(user, container), container);
             transaction.addCommitTask(() -> ListManager.get().addAuditEvent(_list, user, "Deleted " + result + " rows from list."), DbScope.CommitTaskOption.POSTCOMMIT);
             transaction.commit();