Skip to content

Commit 93b6b3f

Browse files
Fix non-deterministic archive output across runs and platforms
Three sources of non-determinism: 1. Directory.GetFiles returns files in filesystem-dependent order. Sort by normalized relative paths (forward slashes, ordinal comparison) so file IDs are assigned identically on any OS. 2. ConcurrentDictionary iteration order is non-deterministic. Sort files by ID in SaveAsync and SaveConcurrentAsync (MS2Lib submodule change). 3. Text files with CRLF line endings produce different archive bytes on Windows vs Linux. Strip all CR bytes from non-binary files before archiving. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 81e1150 commit 93b6b3f

2 files changed

Lines changed: 24 additions & 5 deletions

File tree

MS2Create/Program.cs

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,14 +91,31 @@ private static void AddAndCreateFileToArchive(IMS2Archive archive, (string fullP
9191
(string filePath, string relativePath) = filePaths[index];
9292

9393
uint id = index + 1;
94-
FileStream fsFile = File.OpenRead(filePath);
94+
Stream dataStream = OpenNormalized(filePath);
9595
IMS2FileInfo info = new MS2FileInfo(id.ToString(), relativePath);
96-
IMS2FileHeader header = new MS2FileHeader(fsFile.Length, id, 0, GetCompressionTypeFromFileExtension(filePath));
97-
IMS2File file = new MS2File(archive, fsFile, info, header, false);
96+
IMS2FileHeader header = new MS2FileHeader(dataStream.Length, id, 0, GetCompressionTypeFromFileExtension(filePath));
97+
IMS2File file = new MS2File(archive, dataStream, info, header, false);
9898

9999
archive.Add(file);
100100
}
101101

102+
private static Stream OpenNormalized(string filePath) {
103+
CompressionType ct = GetCompressionTypeFromFileExtension(filePath);
104+
if (ct == CompressionType.Png || ct == CompressionType.Usm) {
105+
return File.OpenRead(filePath);
106+
}
107+
108+
byte[] raw = File.ReadAllBytes(filePath);
109+
int dst = 0;
110+
for (int src = 0; src < raw.Length; src++) {
111+
if (raw[src] != (byte)'\r') {
112+
raw[dst++] = raw[src];
113+
}
114+
}
115+
116+
return new MemoryStream(raw, 0, dst, writable: false);
117+
}
118+
102119
private static (string FullPath, string RelativePath)[] GetFilesRelative(string path) {
103120
if (!path.EndsWith(Path.DirectorySeparatorChar)) {
104121
path += Path.DirectorySeparatorChar;
@@ -108,9 +125,11 @@ private static (string FullPath, string RelativePath)[] GetFilesRelative(string
108125
var result = new (string FullPath, string RelativePath)[files.Length];
109126

110127
for (int i = 0; i < files.Length; i++) {
111-
result[i] = (files[i], files[i].Remove(path));
128+
result[i] = (files[i], files[i].Remove(path).Replace('\\', '/'));
112129
}
113130

131+
Array.Sort(result, (a, b) => StringComparer.Ordinal.Compare(a.RelativePath, b.RelativePath));
132+
114133
return result;
115134
}
116135

MS2Lib

0 commit comments

Comments
 (0)