Consolidate Dataclass data update methods - use DIB for update only

api/src/org/labkey/api/exp/api/ExperimentService.java

@@ -130,6 +130,8 @@ public interface ExperimentService extends ExperimentRunTypeSource
 
     String EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS = "org.labkey.api.exp.api.ExperimentService#FROM_EXPANCESTORS";
 
+    String EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_MERGE = "org.labkey.experiment.api.SampleTypeUpdateServiceDI#ALLOW_ROW_ID_SAMPLE_MERGE";
+
     int SIMPLE_PROTOCOL_FIRST_STEP_SEQUENCE = 1;
     int SIMPLE_PROTOCOL_CORE_STEP_SEQUENCE = 10;
     int SIMPLE_PROTOCOL_EXTRA_STEP_SEQUENCE = 15;
@@ -149,7 +151,6 @@ static void setInstance(ExperimentService impl)
 
     enum QueryOptions
     {
-        UseLsidForUpdate,
         GetSampleRecomputeCol,
         SkipBulkRemapCache,
         DeferRequiredLineageValidation,

api/src/org/labkey/api/exp/query/ExpDataTable.java

@@ -27,42 +27,47 @@ public interface ExpDataTable extends ExpTable<ExpDataTable.Column>
 {
     enum Column
     {
-        RowId,
+        Alias,
+        ContentLink,
+        ClassId, // database table only
+        CpasType, // database table only
+        Created,
+        CreatedBy,
+        DataClass,
+        DataFileUrl,
+        Description,
+        DownloadLink,
+        FileExtension,
+        FileExists,
+        FileSize,
+        Flag,
+        Folder,
+        Generated,
+        InlineThumbnail,
+        Inputs,
+        LastIndexed,
         LSID,
+        Modified,
+        ModifiedBy,
         Name,
-        Description,
-        DataClass,
+        ObjectId, // database table only
+        Outputs,
+        Properties,
         Protocol,
-        SourceProtocolApplication,
-        SourceApplicationInput,
-        DataFileUrl,
         ReferenceCount,
         Run,
         RunApplication,
         RunApplicationOutput,
-        Created,
-        CreatedBy,
-        Modified,
-        ModifiedBy,
-        Folder,
-        Flag,
-        Alias,
-        DownloadLink,
-        ContentLink,
-        ViewFileLink,
+        RunId, // database table only
+        RowId,
+        SourceApplicationId, // database table only
+        SourceApplicationInput,
+        SourceProtocolApplication,
         Thumbnail,
-        InlineThumbnail,
-        FileSize,
-        FileExists,
-        FileExtension,
+        ViewFileLink,
         ViewOrDownload,
         WebDavUrl,
-        WebDavUrlRelative,
-        Generated,
-        LastIndexed,
-        Inputs,
-        Outputs,
-        Properties;
+        WebDavUrlRelative;
 
         public FieldKey fieldKey()
         {

api/src/org/labkey/api/query/AbstractQueryUpdateService.java

@@ -16,6 +16,7 @@
 package org.labkey.api.query;
 
 import org.apache.commons.beanutils.ConversionException;
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.jetbrains.annotations.NotNull;
@@ -122,6 +123,8 @@
 import static org.labkey.api.audit.TransactionAuditProvider.DB_SEQUENCE_NAME;
 import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior;
 import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment;
+import static org.labkey.api.exp.query.ExpMaterialTable.Column.Name;
+import static org.labkey.api.exp.query.ExpMaterialTable.Column.RowId;
 import static org.labkey.api.files.FileContentService.UPLOADED_FILE;
 import static org.labkey.api.util.FileUtil.toFileForRead;
 import static org.labkey.api.util.FileUtil.toFileForWrite;
@@ -453,6 +456,9 @@ protected int _pump(DataIteratorBuilder etl, final @Nullable ArrayList<Map<Strin
     {
         DataIterator it = etl.getDataIterator(context);
 
+        if (null == it)
+            return 0;
+
         try
         {
             if (null != rows)
@@ -900,6 +906,102 @@ public List<Map<String, Object>> updateRows(User user, Container container, List
         return result;
     }
 
+    protected void validatePartitionedRowKeys(Collection<String> columns)
+    {
+        // do nothing
+    }
+
+    public List<Map<String, Object>> updateRowsUsingPartitionedDIB(
+            DbScope.Transaction tx,
+            User user,
+            Container container,
+            List<Map<String, Object>> rows,
+            BatchValidationException errors,
+            @Nullable Map<Enum, Object> configParameters,
+            Map<String, Object> extraScriptContext
+    )
+    {
+        int index = 0;
+        int numPartitions = 0;
+        List<Map<String, Object>> ret = new ArrayList<>();
+
+        Set<Long> observedRowIds = new HashSet<>();
+        Set<String> observedNames = new CaseInsensitiveHashSet();
+
+        while (index < rows.size())
+        {
+            CaseInsensitiveHashSet rowKeys = new CaseInsensitiveHashSet(rows.get(index).keySet());
+
+            validatePartitionedRowKeys(rowKeys);
+
+            int nextIndex = index + 1;
+            while (nextIndex < rows.size() && rowKeys.equals(new CaseInsensitiveHashSet(rows.get(nextIndex).keySet())))
+                nextIndex++;
+
+            List<Map<String, Object>> rowsToProcess = rows.subList(index, nextIndex);
+            index = nextIndex;
+            numPartitions++;
+
+            DataIteratorContext context = getDataIteratorContext(errors, InsertOption.UPDATE, configParameters);
+
+            // skip audit summary for the partitions, we will perform it once at the end
+            context.putConfigParameter(ConfigParameters.SkipAuditSummary, true);
+
+            List<Map<String, Object>> subRet = _updateRowsUsingDIB(user, container, rowsToProcess, context, extraScriptContext);
+
+            // we need to throw if we don't want executeWithRetry() attempt commit()
+            if (context.getErrors().hasErrors())
+                throw new DbScope.RetryPassthroughException(context.getErrors());
+
+            if (subRet != null)
+            {
+                ret.addAll(subRet);
+
+                // Check if duplicate rows have been processed across the partitions
+                // Only start checking for duplicates after the first partition has been processed.
+                if (numPartitions > 1)
+                {
+                    // If we are on the second partition, then lazily check all previous rows, otherwise check only the current partition
+                    checkPartitionForDuplicates(numPartitions == 2 ? ret : subRet, observedRowIds, observedNames, errors);
+                }
+
+                if (errors.hasErrors())
+                    throw new DbScope.RetryPassthroughException(errors);
+            }
+        }
+
+        if (numPartitions > 1)
+        {
+            var auditEvent = tx.getAuditEvent();
+            if (auditEvent != null)
+                auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.DataIteratorPartitions, numPartitions);
+        }
+
+        _addSummaryAuditEvent(container, user, getDataIteratorContext(errors, InsertOption.UPDATE, configParameters), ret.size());
+
+        return ret;
+    }
+
+    private void checkPartitionForDuplicates(List<Map<String, Object>> partitionRows, Set<Long> globalRowIds, Set<String> globalNames, BatchValidationException errors)
+    {
+        for (Map<String, Object> row : partitionRows)
+        {
+            Long rowId = MapUtils.getLong(row, RowId.name());
+            if (rowId != null && !globalRowIds.add(rowId))
+            {
+                errors.addRowError(new ValidationException("Duplicate key provided: " + rowId));
+                return;
+            }
+
+            Object nameObj = row.get(Name.name());
+            if (nameObj != null && !globalNames.add(nameObj.toString()))
+            {
+                errors.addRowError(new ValidationException("Duplicate key provided: " + nameObj));
+                return;
+            }
+        }
+    }
+
     protected void checkDuplicateUpdate(Object pkVals) throws ValidationException
     {
         if (pkVals == null)

api/src/org/labkey/api/query/DefaultQueryUpdateService.java

@@ -778,14 +778,9 @@ protected boolean validMissingValue(Container c, String mv)
         return _missingValues.containsKey(mv);
     }
 
-    protected TableInfo getTableInfoForConversion()
-    {
-        return getDbTable();
-    }
-
     final protected void convertTypes(User user, Container c, Map<String,Object> row) throws ValidationException
     {
-        convertTypes(user, c, row,  getTableInfoForConversion(), null);
+        convertTypes(user, c, row,  getDbTable(), null);
     }
 
     // TODO Path->FileObject

experiment/resources/schemas/dbscripts/postgresql/exp-26.004-26.005.sql

@@ -0,0 +1 @@
+SELECT core.executeJavaUpgradeCode('dropProvisionedDataClassLsidColumn');

experiment/resources/schemas/dbscripts/sqlserver/exp-26.004-26.005.sql

@@ -0,0 +1 @@
+EXEC core.executeJavaUpgradeCode 'dropProvisionedDataClassLsidColumn';