Skip to content

Commit 69eea87

Browse files
ghallidayghalliday
committed
Merge remote-tracking branch 'origin/candidate-9.14.x' into candidate-10.0.x
Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com> # Conflicts: # helm/hpcc/Chart.yaml # helm/hpcc/templates/_helpers.tpl # version.cmake
2 parents 94c601d + 440f9ab commit 69eea87

7 files changed

Lines changed: 167 additions & 11 deletions

File tree

‎dali/base/dadfs.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5188,8 +5188,9 @@ protected: friend class CDistributedFilePart;
51885188
StringBuffer newfn;
51895189
newrfn.getRemotePath(newfn);
51905190
Owned<IFile> f = createIFile(oldrfn);
5191+
Owned<IFile> destFile = createIFile(newrfn);
51915192
if (!isrep||f->exists()) { // ignore non-existant replicates
5192-
f->move(newfn.str());
5193+
f->move(destFile->queryFilename());
51935194
PROGLOG("Succeeded rename %s to %s",oldfn.str(),newfn.str());
51945195
}
51955196
done = true;

‎dali/base/dadiags.cpp‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,9 +195,18 @@ class CDaliDiagnosticsServer: public IDaliServer, public Thread
195195
if (success)
196196
mb.append(connectionInfo);
197197
}
198-
else if (0 == stricmp(id, "save")) {
198+
else if (0 == stricmp(id, "save"))
199+
{
199200
PROGLOG("Dalidiag requests SDS save");
200-
querySDSServer().saveRequest();
201+
try
202+
{
203+
querySDSServer().saveRequest();
204+
}
205+
catch (IException *e)
206+
{
207+
serializeException(e, mb);
208+
e->Release();
209+
}
201210
}
202211
else if (0 == stricmp(id, "settracetransactions")) {
203212
PROGLOG("Dalidiag requests Trace Transactions");

‎dali/base/dasds.cpp‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2108,7 +2108,7 @@ class CCovenSDSManager : public CSDSManagerBase, implements ISDSManagerServer, i
21082108
CheckedCriticalSection treeRegCrit;
21092109
Owned<Thread> unhandledThread;
21102110
unsigned writeTransactions;
2111-
bool ignoreExternals;
2111+
std::atomic<bool> ignoreExternals;
21122112
StringAttr dataPath;
21132113
StringAttr daliName;
21142114
Owned<IPropertyTree> properties;
@@ -6829,11 +6829,6 @@ void CCovenSDSManager::loadStore(const bool *abort)
68296829

68306830
void CCovenSDSManager::saveStore(const char *storeName, SaveStoreFlags flags)
68316831
{
6832-
struct CIgnore
6833-
{
6834-
CIgnore() { SDSManager->ignoreExternals=true; }
6835-
~CIgnore() { SDSManager->ignoreExternals=false; }
6836-
} ignore;
68376832
if (hasMask(flags, ssf_stop))
68386833
{
68396834
// Dali is stopping, we don't care about any inflight transactions any more, we're about to save and quit
@@ -6845,6 +6840,14 @@ void CCovenSDSManager::saveStore(const char *storeName, SaveStoreFlags flags)
68456840
// lock blockedSaveCrit after flush, since deltaWriter itself blocks on blockedSaveCrit
68466841
deltaWriter.flush(); // transactions will be blocked at this stage, no new deltas will be added to writer
68476842
}
6843+
6844+
if (ignoreExternals.exchange(true)) // NB: This is set during save to prevent serialization of externals during the save
6845+
{
6846+
Owned<IException> exception = makeStringException(0, "Save already in progress");
6847+
EXCLOG(exception);
6848+
throw exception.getClear();
6849+
}
6850+
COnScopeExit resetIgnoreExternals([&]() { ignoreExternals = false; });
68486851
CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
68496852
iStoreHelper->saveStore(root, NULL);
68506853
unsigned initNodeTableSize = allNodes.maxElements()+OVERFLOWSIZE;

‎dali/base/dasds.ipp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,10 @@ public:
427427
int errorCode() const { return errCode; }
428428
StringBuffer &errorMessage(StringBuffer &out) const
429429
{
430-
return translateCode(out).append("\n").append(errMsg.str());
430+
translateCode(out);
431+
if (errMsg.length())
432+
out.append(" - ").append(errMsg.str());
433+
return out;
431434
}
432435
MessageAudience errorAudience() const { return MSGAUD_user; }
433436

‎dali/dalidiag/dalidiag.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -640,7 +640,16 @@ int main(int _argc, char* argv[])
640640
MemoryBuffer mb;
641641
mb.append("save");
642642
getDaliDiagnosticValue(mb);
643-
PROGLOG("SDS store saved");
643+
Owned<IException> exception;
644+
if (mb.length())
645+
exception.setown(deserializeException(mb));
646+
if (exception)
647+
{
648+
StringBuffer errMsg;
649+
PROGLOG("Dali SDS save failed: [%d, %s]", exception->errorCode(), exception->errorMessage(errMsg).str());
650+
}
651+
else
652+
PROGLOG("SDS store saved");
644653
break;
645654
}
646655
if (0 == stricmp(arg, "locks")) {

‎testing/regress/ecl/indexfilepos.ecl‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*##############################################################################
2+
3+
HPCC SYSTEMS software Copyright (C) 2025 HPCC Systems®.
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
############################################################################## */
17+
18+
//Test the different variants of fileposition fields
19+
// - fixed and variable with records
20+
// - 3 different index types
21+
// - 2 different ways of defining the index (from dataset or from record)
22+
23+
//version compressionType='legacy',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=false
24+
//version compressionType='legacy',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=false
25+
//version compressionType='legacy',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=false
26+
//version compressionType='legacy',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=false
27+
//version compressionType='legacy',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=false
28+
//version compressionType='legacy',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=false
29+
30+
//version compressionType='legacy',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=true
31+
//version compressionType='legacy',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=true
32+
//version compressionType='legacy',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=true
33+
//version compressionType='legacy',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=true
34+
//version compressionType='legacy',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=true
35+
//version compressionType='legacy',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=true
36+
37+
//version compressionType='inplace',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=false
38+
//version compressionType='inplace',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=false
39+
//version compressionType='inplace',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=false
40+
//version compressionType='inplace',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=false
41+
//version compressionType='inplace',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=false
42+
//version compressionType='inplace',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=false
43+
44+
//version compressionType='inplace',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=true
45+
//version compressionType='inplace',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=true
46+
//version compressionType='inplace',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=true
47+
//version compressionType='inplace',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=true
48+
//version compressionType='inplace',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=true
49+
//version compressionType='inplace',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=true
50+
51+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=false
52+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=false
53+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=false
54+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=false
55+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=false
56+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=false
57+
58+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=false,hasFilePos=false,fromDataset=true
59+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=false,hasFilePos=true,fromDataset=true
60+
//version compressionType='hybrid',variableWidth=false,zeroFilePos=true,hasFilePos=true,fromDataset=true
61+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=false,hasFilePos=false,fromDataset=true
62+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=false,hasFilePos=true,fromDataset=true
63+
//version compressionType='hybrid',variableWidth=true,zeroFilePos=true,hasFilePos=true,fromDataset=true
64+
65+
import ^ as root;
66+
import $.setup;
67+
import Std.File AS FileServices;
68+
69+
compressionType := #IFDEFINED(root.compressionType, 'hybrid');
70+
variableWidth := #IFDEFINED(root.variableWidth, true);
71+
zeroFilePos := #IFDEFINED(root.zeroFilePos, true);
72+
hasFilePos := #IFDEFINED(root.hasFilePos, true);
73+
fromDataset := #IFDEFINED(root.fromDataset, true);
74+
75+
idxRecord := RECORD
76+
unsigned uid
77+
=>
78+
unsigned uid2;
79+
#if (variableWidth)
80+
string pad := '<pad>';
81+
#end
82+
#if (not zeroFilePos)
83+
unsigned8 uid3;
84+
#end
85+
END;
86+
87+
numParents := 50;
88+
89+
ds := DATASET(numParents,
90+
TRANSFORM(idxRecord,
91+
SELF.uid := COUNTER;
92+
SELF.uid2 := HASH32(COUNTER);
93+
#if (variableWidth)
94+
SELF.pad := 'Arfle Barfle Gloop!!'[1..COUNTER%20];
95+
#end
96+
#if (not zeroFilePos)
97+
SELF.uid3 := COUNTER;
98+
#end
99+
), DISTRIBUTED);
100+
101+
102+
prefix := setup.Files(false, false).indexPrefix + WORKUNIT;
103+
indexName := prefix+'_fpos_index';
104+
105+
#if (fromDataset)
106+
i := INDEX(ds, { uid }, {ds}, indexName, compressed(compressionType),FILEPOSITION(hasFilePos));
107+
#else
108+
i := INDEX(idxRecord, indexName, compressed(compressionType),FILEPOSITION(hasFilePos));
109+
#end
110+
111+
// NOTE: use recordof(i) in the following to ensure that any implicit file position is included
112+
lhs := PROJECT(SORT(ds, uid2), TRANSFORM(recordof(i), SELF := LEFT));
113+
114+
matchRecord := RECORD
115+
recordof(i) l;
116+
recordof(i) r;
117+
END;
118+
119+
j := JOIN(lhs, i, LEFT.uid=RIGHT.uid, TRANSFORM(matchRecord, SELF.l := LEFT; SELF.r := RIGHT), KEEP(1));
120+
121+
SEQUENTIAL(
122+
BUILD(i, ds, OVERWRITE);
123+
OUTPUT(j(l != r));
124+
125+
// Clean-up
126+
FileServices.DeleteLogicalFile(indexName)
127+
);

‎testing/regress/ecl/key/indexfilepos.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<Dataset name='Result 1'>
2+
</Dataset>
3+
<Dataset name='Result 2'>
4+
</Dataset>

0 commit comments

Comments
 (0)