Integrate RemoteState and MaxAge for AbstractProvider

andreaskueffel · andreaskueffel · commit 1268ce6d70d1 · 2025-01-13T11:13:36.000+01:00
diff --git a/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs b/FileSyncLibNet/AccessProviders/FileIoAccessProvider.cs
@@ -10,10 +10,14 @@ namespace FileSyncLibNet.AccessProviders
     internal class FileIoAccessProvider : IAccessProvider
     {
         public string AccessPath { get; private set; }
+        private readonly RemoteState remoteState;
         private readonly ILogger logger;
-        public FileIoAccessProvider(ILogger logger)
+        public FileIoAccessProvider(ILogger logger, string stateFilename)
         {
             this.logger = logger;
+            if (!string.IsNullOrEmpty(stateFilename))
+                remoteState = new RemoteState(stateFilename);
+            
         }
         public void UpdateAccessPath(string accessPath)
         {
@@ -27,6 +31,9 @@ public void CreateDirectory(string path)
 
         public FileInfo2 GetFileInfo(string path)
         {
+            if(null!=remoteState)
+                return remoteState.GetFileInfo(Path.Combine(AccessPath, path));
+
             var fi = new FileInfo(Path.Combine(AccessPath, path));
 
             return new FileInfo2(path, fi.Exists)
@@ -75,12 +82,13 @@ public void WriteFile(FileInfo2 file, Stream content)
                 content.CopyTo(stream);
             }
             File.SetLastWriteTime(realFilename, file.LastWriteTime);
-
+            remoteState?.SetFileInfo(realFilename, file);
         }
         public void Delete(FileInfo2 fileInfo)
         {
             var realFilename = Path.Combine(AccessPath, fileInfo.Name);
             File.Delete(realFilename);
+            remoteState?.RemoveFileInfo(realFilename);
         }
     }
 }
diff --git a/FileSyncLibNet/AccessProviders/ScpAccessProvider.cs b/FileSyncLibNet/AccessProviders/ScpAccessProvider.cs
@@ -15,10 +15,14 @@ internal class ScpAccessProvider : IAccessProvider
         public string AccessPathUri { get; private set; }
         NetworkCredential Credentials { get; }
         SftpClient ftpClient;
+        private readonly RemoteState remoteState;
         public string AccessPath { get; private set; }
+        
         private readonly ILogger logger;
-        public ScpAccessProvider(NetworkCredential credentials, ILogger logger)
+        public ScpAccessProvider(NetworkCredential credentials, ILogger logger, string stateFilename)
         {
+            if (!string.IsNullOrEmpty(stateFilename))
+                remoteState = new RemoteState(stateFilename);
             this.logger = logger;
             Credentials = credentials;
         }
@@ -101,8 +105,11 @@ public void CreateDirectory(string path)
 
         public FileInfo2 GetFileInfo(string path)
         {
-            EnsureConnected();
             var realFilename = System.IO.Path.Combine(AccessPath, path).Replace("\\", "/");
+            if (remoteState != null)
+                return remoteState.GetFileInfo(realFilename);
+
+            EnsureConnected();
             if (ftpClient.Exists(realFilename))
             {
                 var fi = ftpClient.ListDirectory(realFilename);
@@ -149,6 +156,7 @@ public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, s
                 try
                 {
                     var files = ftpClient.ListDirectory(basePath);
+                    var sepChar="/";
                     if (recursive)
                     {
                         foreach (var folder in files.Where(x => x.IsDirectory && !(x.Name == ".") && !(x.Name == "..")))
@@ -158,7 +166,7 @@ public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, s
                         }
                     }
                     ret_val.AddRange(files.Where(x => MatchesPattern(x.Name, pattern)).Where(x => x.LastWriteTime >= minimumLastWriteTime && !x.IsDirectory).Select(x =>
-                    new FileInfo2($"{basePath.Substring(AccessPath.Length + 1)}/{x.Name}", exists: true) { LastWriteTime = x.LastWriteTime, Length = x.Length }).ToList());
+                    new FileInfo2($"{(AccessPath.Length + 1<basePath.Length?(basePath.Substring(AccessPath.Length+1))+sepChar:string.Empty)}{x.Name}", exists: true) { LastWriteTime = x.LastWriteTime, Length = x.Length }).ToList());
                 }
                 catch (Exception exc)
                 {
@@ -188,13 +196,15 @@ public void WriteFile(FileInfo2 file, Stream content)
 
                 content.CopyTo(stream);
             }
+            remoteState?.SetFileInfo(filePath, file);
         }
 
         public void Delete(FileInfo2 fileInfo)
         {
             EnsureConnected();
             var filePath = System.IO.Path.Combine(AccessPath, fileInfo.Name).Replace("\\", "/");
             ftpClient.Delete(filePath);
+            remoteState?.RemoveFileInfo(filePath);
         }
     }
 }
diff --git a/FileSyncLibNet/Commons/FileJobOptionsBase.cs b/FileSyncLibNet/Commons/FileJobOptionsBase.cs
@@ -3,6 +3,7 @@
 using Microsoft.Extensions.Logging;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Net;
 
 namespace FileSyncLibNet.FileCleanJob
@@ -17,5 +18,19 @@ public abstract class FileJobOptionsBase : IFileJobOptions
         public List<string> Subfolders { get; set; } = new List<string>();
         public bool Recursive { get; set; } = true;
         public SyncProvider FileSyncProvider { get; set; } = SyncProvider.FileIO;
+
+        public virtual string GetHashedName()
+        {
+            string readableInfo = $"{Path.GetFileName(DestinationPath.TrimEnd(Path.DirectorySeparatorChar))}_{Interval.TotalMinutes}min";
+            string allProperties = $"{DestinationPath}_{SearchPattern}_{Interval}_{Recursive}_{string.Join(",", Subfolders)}";
+            string hash;
+            using (var sha256 = System.Security.Cryptography.SHA256.Create())
+            {
+                byte[] bytes = System.Text.Encoding.UTF8.GetBytes(allProperties);
+                byte[] hashBytes = sha256.ComputeHash(bytes);
+                hash = BitConverter.ToString(hashBytes).Replace("-", "").ToLower();
+            }
+            return $"{readableInfo}_{hash}";
+        }
     }
 }
diff --git a/FileSyncLibNet/Commons/IFileJobOptions.cs b/FileSyncLibNet/Commons/IFileJobOptions.cs
@@ -17,5 +17,6 @@ public interface IFileJobOptions
         bool Recursive { get; set; }
         string SearchPattern { get; set; }
         List<string> Subfolders { get; set; }
+        string GetHashedName();
     }
 }
diff --git a/FileSyncLibNet/SyncProviders/AbstractProvider.cs b/FileSyncLibNet/SyncProviders/AbstractProvider.cs
@@ -18,20 +18,21 @@ public AbstractProvider(IFileJobOptions jobOptions) : base(jobOptions)
                 throw new ArgumentException("this instance has no information about syncing files, it has type " + JobOptions.GetType().ToString());
             if (syncJobOptions.SourcePath.StartsWith("scp:"))
             {
-                SourceAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger);
+                SourceAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger, stateFilename: null);
             }
             else
             {
-                SourceAccess = new FileIoAccessProvider(jobOptions.Logger);
+                SourceAccess = new FileIoAccessProvider(jobOptions.Logger, stateFilename: null);
             }
             SourceAccess.UpdateAccessPath(syncJobOptions.SourcePath);
+            string stateFilename = syncJobOptions.RememberRemoteState ? syncJobOptions.GetHashedName() : null;
             if (syncJobOptions.DestinationPath.StartsWith("scp:"))
             {
-                DestinationAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger);
+                DestinationAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger, stateFilename);
             }
             else
             {
-                DestinationAccess = new FileIoAccessProvider(jobOptions.Logger);
+                DestinationAccess = new FileIoAccessProvider(jobOptions.Logger, stateFilename);
             }
 
         }
@@ -57,7 +58,13 @@ public override void SyncSourceToDest()
             bool createDestinationDir = true;
             int copied = 0;
             int skipped = 0;
-            var minimumLastWriteTime = jobOptions.RememberLastSync ? (LastRun - jobOptions.Interval - jobOptions.Interval) : DateTime.MinValue;
+            var minimumLastWriteTime = jobOptions.RememberLastSync ?
+                (LastRun - jobOptions.Interval - jobOptions.Interval) :
+                (jobOptions.MaxAge < jobOptions.Interval ?
+                    DateTimeOffset.MinValue :
+                    DateTimeOffset.Now - jobOptions.MaxAge
+                );
+
             bool error_occured = false;
             try
             {

Original file line number	Diff line number	Diff line change
`@@ -10,10 +10,14 @@ namespace FileSyncLibNet.AccessProviders`
`10`	`10`	`internal class FileIoAccessProvider : IAccessProvider`
`11`	`11`	`{`
`12`	`12`	`public string AccessPath { get; private set; }`
	`13`	`+ private readonly RemoteState remoteState;`
`13`	`14`	`private readonly ILogger logger;`
`14`		`- public FileIoAccessProvider(ILogger logger)`
	`15`	`+ public FileIoAccessProvider(ILogger logger, string stateFilename)`
`15`	`16`	`{`
`16`	`17`	`this.logger = logger;`
	`18`	`+ if (!string.IsNullOrEmpty(stateFilename))`
	`19`	`+ remoteState = new RemoteState(stateFilename);`
	`20`	`+`
`17`	`21`	`}`
`18`	`22`	`public void UpdateAccessPath(string accessPath)`
`19`	`23`	`{`
`@@ -27,6 +31,9 @@ public void CreateDirectory(string path)`
`27`	`31`
`28`	`32`	`public FileInfo2 GetFileInfo(string path)`
`29`	`33`	`{`
	`34`	`+ if(null!=remoteState)`
	`35`	`+ return remoteState.GetFileInfo(Path.Combine(AccessPath, path));`
	`36`	`+`
`30`	`37`	`var fi = new FileInfo(Path.Combine(AccessPath, path));`
`31`	`38`
`32`	`39`	`return new FileInfo2(path, fi.Exists)`
`@@ -75,12 +82,13 @@ public void WriteFile(FileInfo2 file, Stream content)`
`75`	`82`	`content.CopyTo(stream);`
`76`	`83`	`}`
`77`	`84`	`File.SetLastWriteTime(realFilename, file.LastWriteTime);`
`78`		`-`
	`85`	`+ remoteState?.SetFileInfo(realFilename, file);`
`79`	`86`	`}`
`80`	`87`	`public void Delete(FileInfo2 fileInfo)`
`81`	`88`	`{`
`82`	`89`	`var realFilename = Path.Combine(AccessPath, fileInfo.Name);`
`83`	`90`	`File.Delete(realFilename);`
	`91`	`+ remoteState?.RemoveFileInfo(realFilename);`
`84`	`92`	`}`
`85`	`93`	`}`
`86`	`94`	`}`
Original file line number	Diff line number	Diff line change
`@@ -15,10 +15,14 @@ internal class ScpAccessProvider : IAccessProvider`
`15`	`15`	`public string AccessPathUri { get; private set; }`
`16`	`16`	`NetworkCredential Credentials { get; }`
`17`	`17`	`SftpClient ftpClient;`
	`18`	`+ private readonly RemoteState remoteState;`
`18`	`19`	`public string AccessPath { get; private set; }`
	`20`	`+`
`19`	`21`	`private readonly ILogger logger;`
`20`		`- public ScpAccessProvider(NetworkCredential credentials, ILogger logger)`
	`22`	`+ public ScpAccessProvider(NetworkCredential credentials, ILogger logger, string stateFilename)`
`21`	`23`	`{`
	`24`	`+ if (!string.IsNullOrEmpty(stateFilename))`
	`25`	`+ remoteState = new RemoteState(stateFilename);`
`22`	`26`	`this.logger = logger;`
`23`	`27`	`Credentials = credentials;`
`24`	`28`	`}`
`@@ -101,8 +105,11 @@ public void CreateDirectory(string path)`
`101`	`105`
`102`	`106`	`public FileInfo2 GetFileInfo(string path)`
`103`	`107`	`{`
`104`		`- EnsureConnected();`
`105`	`108`	`var realFilename = System.IO.Path.Combine(AccessPath, path).Replace("\\", "/");`
	`109`	`+ if (remoteState != null)`
	`110`	`+ return remoteState.GetFileInfo(realFilename);`
	`111`	`+`
	`112`	`+ EnsureConnected();`
`106`	`113`	`if (ftpClient.Exists(realFilename))`
`107`	`114`	`{`
`108`	`115`	`var fi = ftpClient.ListDirectory(realFilename);`
`@@ -149,6 +156,7 @@ public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, s`
`149`	`156`	`try`
`150`	`157`	`{`
`151`	`158`	`var files = ftpClient.ListDirectory(basePath);`
	`159`	`+ var sepChar="/";`
`152`	`160`	`if (recursive)`
`153`	`161`	`{`
`154`	`162`	`foreach (var folder in files.Where(x => x.IsDirectory && !(x.Name == ".") && !(x.Name == "..")))`
`@@ -158,7 +166,7 @@ public List<FileInfo2> GetFiles(DateTime minimumLastWriteTime, string pattern, s`
`158`	`166`	`}`
`159`	`167`	`}`
`160`	`168`	`ret_val.AddRange(files.Where(x => MatchesPattern(x.Name, pattern)).Where(x => x.LastWriteTime >= minimumLastWriteTime && !x.IsDirectory).Select(x =>`
`161`		`- new FileInfo2($"{basePath.Substring(AccessPath.Length + 1)}/{x.Name}", exists: true) { LastWriteTime = x.LastWriteTime, Length = x.Length }).ToList());`
	`169`	`+ new FileInfo2($"{(AccessPath.Length + 1<basePath.Length?(basePath.Substring(AccessPath.Length+1))+sepChar:string.Empty)}{x.Name}", exists: true) { LastWriteTime = x.LastWriteTime, Length = x.Length }).ToList());`
`162`	`170`	`}`
`163`	`171`	`catch (Exception exc)`
`164`	`172`	`{`
`@@ -188,13 +196,15 @@ public void WriteFile(FileInfo2 file, Stream content)`
`188`	`196`
`189`	`197`	`content.CopyTo(stream);`
`190`	`198`	`}`
	`199`	`+ remoteState?.SetFileInfo(filePath, file);`
`191`	`200`	`}`
`192`	`201`
`193`	`202`	`public void Delete(FileInfo2 fileInfo)`
`194`	`203`	`{`
`195`	`204`	`EnsureConnected();`
`196`	`205`	`var filePath = System.IO.Path.Combine(AccessPath, fileInfo.Name).Replace("\\", "/");`
`197`	`206`	`ftpClient.Delete(filePath);`
	`207`	`+ remoteState?.RemoveFileInfo(filePath);`
`198`	`208`	`}`
`199`	`209`	`}`
`200`	`210`	`}`
Original file line number	Diff line number	Diff line change
`@@ -17,5 +17,6 @@ public interface IFileJobOptions`
`17`	`17`	`bool Recursive { get; set; }`
`18`	`18`	`string SearchPattern { get; set; }`
`19`	`19`	`List<string> Subfolders { get; set; }`
	`20`	`+ string GetHashedName();`
`20`	`21`	`}`
`21`	`22`	`}`
Original file line number	Diff line number	Diff line change
`@@ -18,20 +18,21 @@ public AbstractProvider(IFileJobOptions jobOptions) : base(jobOptions)`
`18`	`18`	`throw new ArgumentException("this instance has no information about syncing files, it has type " + JobOptions.GetType().ToString());`
`19`	`19`	`if (syncJobOptions.SourcePath.StartsWith("scp:"))`
`20`	`20`	`{`
`21`		`- SourceAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger);`
	`21`	`+ SourceAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger, stateFilename: null);`
`22`	`22`	`}`
`23`	`23`	`else`
`24`	`24`	`{`
`25`		`- SourceAccess = new FileIoAccessProvider(jobOptions.Logger);`
	`25`	`+ SourceAccess = new FileIoAccessProvider(jobOptions.Logger, stateFilename: null);`
`26`	`26`	`}`
`27`	`27`	`SourceAccess.UpdateAccessPath(syncJobOptions.SourcePath);`
	`28`	`+ string stateFilename = syncJobOptions.RememberRemoteState ? syncJobOptions.GetHashedName() : null;`
`28`	`29`	`if (syncJobOptions.DestinationPath.StartsWith("scp:"))`
`29`	`30`	`{`
`30`		`- DestinationAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger);`
	`31`	`+ DestinationAccess = new ScpAccessProvider(syncJobOptions.Credentials, jobOptions.Logger, stateFilename);`
`31`	`32`	`}`
`32`	`33`	`else`
`33`	`34`	`{`
`34`		`- DestinationAccess = new FileIoAccessProvider(jobOptions.Logger);`
	`35`	`+ DestinationAccess = new FileIoAccessProvider(jobOptions.Logger, stateFilename);`
`35`	`36`	`}`
`36`	`37`
`37`	`38`	`}`
`@@ -57,7 +58,13 @@ public override void SyncSourceToDest()`
`57`	`58`	`bool createDestinationDir = true;`
`58`	`59`	`int copied = 0;`
`59`	`60`	`int skipped = 0;`
`60`		`- var minimumLastWriteTime = jobOptions.RememberLastSync ? (LastRun - jobOptions.Interval - jobOptions.Interval) : DateTime.MinValue;`
	`61`	`+ var minimumLastWriteTime = jobOptions.RememberLastSync ?`
	`62`	`+ (LastRun - jobOptions.Interval - jobOptions.Interval) :`
	`63`	`+ (jobOptions.MaxAge < jobOptions.Interval ?`
	`64`	`+ DateTimeOffset.MinValue :`
	`65`	`+ DateTimeOffset.Now - jobOptions.MaxAge`
	`66`	`+ );`
	`67`	`+`
`61`	`68`	`bool error_occured = false;`
`62`	`69`	`try`
`63`	`70`	`{`