Create new temp file interface

New temporary file handling interface created.
hthor changed to use the new interface.

Signed-off-by: Dave Streeter <dave.streeter@lexisnexisrisk.com>

Author: streeterd

ecl/eclagent/agentctx.hpp

@@ -80,7 +80,14 @@ interface IOrderedOutputSerializer;
 typedef enum { ofSTD, ofXML, ofRAW } outputFmts;
 enum class AccessMode : unsigned;
 
-struct IAgentContext : extends IGlobalCodeContext
+interface ITempFileHandler
+{
+    virtual const char *noteTemporaryFile(const char * fname) = 0;
+    virtual const char *queryTemporaryFile(const char * fname) = 0;
+    virtual void removeTemporaryFile(const char * fname) = 0;
+};
+
+struct IAgentContext : extends IGlobalCodeContext, extends ITempFileHandler
 {
     virtual void reportProgress(const char *msg, unsigned flags=0) = 0;
     virtual bool queryResolveFilesLocally() = 0;
@@ -92,12 +99,9 @@ struct IAgentContext : extends IGlobalCodeContext
 
     virtual IConstWorkUnit *queryWorkUnit() const = 0;
     virtual IWorkUnit *updateWorkUnit() const = 0;
-    
+
     virtual ILocalOrDistributedFile *resolveLFN(const char *logicalName, const char *errorTxt, bool optional, bool noteRead, AccessMode accessMode, StringBuffer * expandedlfn, bool isPrivilegedUser) = 0;
     virtual StringBuffer & getTempfileBase(StringBuffer & buff) = 0;
-    virtual const char *noteTemporaryFile(const char *fname) = 0;
-    virtual const char *noteTemporaryFilespec(const char *fname) = 0;
-    virtual const char *queryTemporaryFile(const char *fname) = 0;
     virtual void reloadWorkUnit() = 0;
 
     virtual char *resolveName(const char *in, char *out, unsigned outlen) = 0;
@@ -110,14 +114,14 @@ struct IAgentContext : extends IGlobalCodeContext
     virtual void outputFormattedResult(const char *name, unsigned sequence, bool close) = 0;
 
     virtual const char *queryAllowedPipePrograms() = 0;
-    
+
     virtual IOrderedOutputSerializer * queryOutputSerializer() = 0;
 
     virtual IGroup *getHThorGroup(StringBuffer &grpnameout) = 0;
 
     virtual RecordTranslationMode getLayoutTranslationMode() const = 0;
     virtual unsigned __int64 queryStopAfter() = 0;
-    
+
     virtual const char *queryWuid() = 0;
 
     virtual void updateWULogfile(IWorkUnit *outputWU) = 0;

ecl/eclagent/eclagent.cpp

@@ -615,46 +615,90 @@ StringBuffer & EclAgent::getTempfileBase(StringBuffer & buff)
     return buff.append(queryTempfilePath()).append(PATHSEPCHAR).appendLower(wuid);
 }
 
-const char *EclAgent::queryTemporaryFile(const char *fname)
+const char *EclAgent::queryTemporaryFile(const char * fname)
 {
-    StringBuffer tempfilename;
-    getTempfileBase(tempfilename).append(PATHSEPCHAR).append(fname);
+    dbgassertex(!isEmptyString(fname));
+
     CriticalBlock crit(tfsect);
-    ForEachItemIn(idx, tempFiles)
-    {
-        if (strcmp(tempFiles.item(idx), tempfilename.str())==0)
-            return tempFiles.item(idx);
-    }
+
+    Linked<CTempFileInfo> tempFileInfo = tempFileInfoMap.getValue(fname);
+    if (tempFileInfo)
+        return tempFileInfo->tempFileName.str();
+
     StringBuffer errmsg;
-    errmsg.append("Attempt to read temp file that has not yet been registered: ").append(tempfilename);
+    errmsg.append("Attempt to read temp file that has not yet been registered: ").append(tempFileInfo->tempFileName.str());
     fail(0, errmsg.str());
-    return 0;
 }
 
-const char *EclAgent::noteTemporaryFile(const char *fname)
+const char *EclAgent::noteTemporaryFile(const char * fname)
 {
-    StringBuffer tempfilename;
-    getTempfileBase(tempfilename).append(PATHSEPCHAR).append(fname);
+    if (isEmptyString(fname))
+        fail(0, "Attempt to register empty temp file name");
+
     CriticalBlock crit(tfsect);
-    tempFiles.append(tempfilename.str());
-    return tempFiles.item(tempFiles.length()-1);
+
+    Linked<CTempFileInfo> tempFileInfo = tempFileInfoMap.getValue(fname);
+    if (tempFileInfo)
+    {
+        WARNLOG("Warning: temp file already registered %s", fname);
+        return tempFileInfo->tempFileName.str();
+    }
+
+    /* tempOwnerId is used at the end of the temporary file name to ensure
+     * that in the event of a crash, we can use the pid and start time to
+     * determine if temp files are from a currently running process or not,
+     * and therefore whether they can be safely deleted.
+     */
+    if (tempOwnerId.isEmpty())
+    {
+        unsigned pid = (unsigned)GetCurrentProcessId();
+        unsigned __int64 startEpoch = (unsigned __int64)time(nullptr);
+        tempOwnerId.append('.').append(pid).append('.').append(startEpoch);
+    }
+
+    tempFileInfo.setown(new CTempFileInfo);
+    tempFileInfoMap.setValue(fname, tempFileInfo.getLink());
+
+    StringBuffer base;
+    getTempfileBase(base);
+    base.append(PATHSEPCHAR);
+    StringBuffer tail;
+    splitFilename(fname, nullptr, nullptr, &tail, nullptr);
+    tempFileInfo->tempFileName.append(base).append(tail).append(tempOwnerId);
+
+    return tempFileInfo->tempFileName.str();
 }
 
-const char *EclAgent::noteTemporaryFilespec(const char *fspec)
+void EclAgent::removeTemporaryFile(const char * fname)
 {
-    CriticalBlock crit(tfsect);
-    tempFiles.append(fspec);
-    return tempFiles.item(tempFiles.length()-1);
+    dbgassertex(!isEmptyString(fname));
+
+    const char *tempFileName = queryTemporaryFile(fname);
+    if (tempFileName)
+    {
+        Owned<IFile> file = createIFile(tempFileName);
+        if (file->isFile() != fileBool::foundYes)
+            return;
+
+        file->remove();
+    }
 }
 
 void EclAgent::deleteTempFiles()
 {
     CriticalBlock crit(tfsect);
-    ForEachItemIn(idx, tempFiles)
+
+    if (!tempOwnerId.isEmpty())
     {
-        remove(tempFiles.item(idx));
+        HashIterator it(tempFileInfoMap);
+        ForEach(it)
+        {
+            const char *originalId = static_cast<const char *>(it.query().getKey());
+            CTempFileInfo *tempFileInfo = tempFileInfoMap.getValue(originalId);
+            if (tempFileInfo)
+                removeTemporaryFile(originalId);
+        }
     }
-    tempFiles.kill();
 }
 
 const char *EclAgent::loadResource(unsigned id)

ecl/eclagent/eclagent.ipp

@@ -172,17 +172,17 @@ public:
     {
         return ctx->getTempfileBase(buff);
     }
-    virtual const char *noteTemporaryFile(const char *fname)
+    virtual const char *noteTemporaryFile(const char * fname)
     {
         return ctx->noteTemporaryFile(fname);
     }
-    virtual const char *noteTemporaryFilespec(const char *fspec)
+    virtual const char *queryTemporaryFile(const char * fname)
     {
-        return ctx->noteTemporaryFilespec(fspec);
+        return ctx->queryTemporaryFile(fname);
     }
-    virtual const char *queryTemporaryFile(const char *fname)
+    virtual void removeTemporaryFile(const char * fname)
     {
-        return ctx->queryTemporaryFile(fname);
+        ctx->removeTemporaryFile(fname);
     }
     virtual void reloadWorkUnit()
     {
@@ -357,6 +357,11 @@ class EclAgent : implements IAgentContext, implements ICodeContext, implements I
 private:
     friend class EclAgentWorkflowMachine;
 
+    struct CTempFileInfo : public CInterfaceOf<IInterface>
+    {
+        StringBuffer tempFileName;
+    };
+
     Owned<EclAgentWorkflowMachine> workflow;
     Owned<IConstWorkUnit> wuRead;
     Owned<roxiemem::IRowManager> rowManager;
@@ -378,7 +383,8 @@ private:
     outputFmts outputFmt = ofSTD;
     unsigned __int64 stopAfter;
     mutable CriticalSection wusect;
-    StringArray tempFiles;
+    MapStringToMyClass<CTempFileInfo> tempFileInfoMap; // Map of original temp file name to the actual temp file name on disk
+    StringBuffer tempOwnerId; // Holds the current pid and start epoch of the process for temp files in map tempFileInfoMap
     CriticalSection tfsect;
     IArray persistReadLocks;
     StringArray processedPersists;
@@ -615,9 +621,9 @@ public:
 
     virtual unsigned __int64 getDatasetHash(const char * name, unsigned __int64 hash);
     virtual void reportProgress(const char *msg, unsigned flags=0);
-    virtual const char *noteTemporaryFile(const char *fname);
-    virtual const char *noteTemporaryFilespec(const char *fspec);
-    virtual const char *queryTemporaryFile(const char *fname);
+    virtual const char *noteTemporaryFile(const char * fname);
+    virtual const char *queryTemporaryFile(const char * fname);
+    virtual void removeTemporaryFile(const char * fname);
     virtual void deleteFile(const char * logicalName);
 
     void addException(ErrorSeverity severity, const char * source, unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool failOnError, bool isAbort);