Replace MemoryMappedFile with FileStream for Linux compatibility

AngeloTadeucci · claude · AngeloTadeucci · commit fc36d2fcae79 · 2026-03-14T21:36:45.000-03:00
MemoryMappedViewStream returns corrupted bytes on Linux for some m2d
entries. Switch to FileStream with Seek+Read and a lock for thread
safety. Also handle Base64URL characters in encrypted data and add
error recovery in NifParser. Fix cross-platform paths in TestUtils.

Co-Authored-By: Claude Opus 4.6 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/Maple2.File.IO/Crypto/CryptoManager.cs b/Maple2.File.IO/Crypto/CryptoManager.cs
@@ -1,5 +1,4 @@
 ﻿using System.IO.Compression;
-using System.IO.MemoryMappedFiles;
 using System.Text;
 using Maple2.File.IO.Crypto.Common;
 using Maple2.File.IO.Crypto.Keys;
@@ -32,12 +31,24 @@ public static byte[] DecryptFileTable(IPackStream stream, System.IO.Stream buffe
         throw new Exception("ERROR decrypting file table: the size of the table is invalid.");
     }
 
-    public static byte[] DecryptData(IPackFileHeader pHeader, MemoryMappedFile data) {
+    public static byte[] DecryptData(IPackFileHeader pHeader, FileStream data, object readLock) {
         if (pHeader.CompressedFileSize > 0 && pHeader.EncodedFileSize > 0 && pHeader.FileSize > 0) {
-            using MemoryMappedViewStream buffer = data.CreateViewStream((long) pHeader.Offset, pHeader.EncodedFileSize);
             byte[] src = new byte[pHeader.EncodedFileSize];
+            int bytesRead;
+            lock (readLock) {
+                data.Seek((long) pHeader.Offset, SeekOrigin.Begin);
+                int totalRead = 0;
+                int remaining = (int) pHeader.EncodedFileSize;
+                while (remaining > 0) {
+                    int read = data.Read(src, totalRead, remaining);
+                    if (read == 0) break;
+                    totalRead += read;
+                    remaining -= read;
+                }
+                bytesRead = totalRead;
+            }
 
-            if (buffer.Read(src, 0, (int) pHeader.EncodedFileSize) == pHeader.EncodedFileSize) {
+            if (bytesRead == pHeader.EncodedFileSize) {
                 return Decrypt(pHeader.Version, pHeader.EncodedFileSize, (uint) pHeader.CompressedFileSize, pHeader.BufferFlag, src);
             }
         }
@@ -47,19 +58,35 @@ public static byte[] DecryptData(IPackFileHeader pHeader, MemoryMappedFile data)
 
     // Decryption Routine: Base64 -> AES -> Zlib
     private static byte[] Decrypt(PackVersion version, uint size, uint sizeCompressed, Encryption flag, byte[] src) {
-        if (flag.HasFlag(Encryption.Aes)) {
+        if (flag.HasFlag(Encryption.Xor)) {
+            // Decrypt the XOR encrypted block (check Xor before Aes because 0xFF is a superset of 0xEE in bits)
+            src = EncryptXor(version, src, size, sizeCompressed);
+        } else if (flag.HasFlag(Encryption.Aes)) {
             // Get the AES Key/IV for transformation
             CipherKeys.GetKeyAndIV(version, sizeCompressed, out byte[] key, out byte[] iv);
 
-            // Decode the base64 encoded string
-            src = Convert.FromBase64String(Encoding.UTF8.GetString(src));
+            // Decode the base64 encoded string (handle both standard and URL-safe Base64)
+            string b64 = Encoding.UTF8.GetString(src).Replace('-', '+').Replace('_', '/');
+            try {
+                src = Convert.FromBase64String(b64);
+            } catch (FormatException) {
+                const string valid = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/= \t\r\n";
+                for (int i = 0; i < b64.Length; i++) {
+                    if (valid.IndexOf(b64[i]) < 0) {
+                        Console.Error.WriteLine($"[Base64 FAIL] Invalid char at {i}/{b64.Length}: 0x{(int)b64[i]:X4} '{b64[i]}'");
+                        Console.Error.WriteLine($"[Base64 FAIL] Context: ...{b64[Math.Max(0,i-8)..Math.Min(b64.Length,i+8)]}...");
+                        break;
+                    }
+                }
+                if (b64.Length % 4 != 0)
+                    Console.Error.WriteLine($"[Base64 FAIL] Length {b64.Length} not multiple of 4 (remainder {b64.Length % 4})");
+                Console.Error.WriteLine($"[Base64 FAIL] flag=0x{(uint)flag:X8} size={size} sizeCompressed={sizeCompressed} srcLen={b64.Length}");
+                throw;
+            }
 
             // Decrypt the AES encrypted block
             var pCipher = new AESCipher(key, iv);
             pCipher.TransformBlock(src, 0, size, src, 0);
-        } else if (flag.HasFlag(Encryption.Xor)) {
-            // Decrypt the XOR encrypted block
-            src = EncryptXor(version, src, size, sizeCompressed);
         }
 
         return flag.HasFlag(Encryption.Zlib) ? UncompressBuffer(src) : src;
diff --git a/Maple2.File.IO/M2dReader.cs b/Maple2.File.IO/M2dReader.cs
@@ -1,5 +1,4 @@
-﻿using System.Globalization;
-using System.IO.MemoryMappedFiles;
+using System.Globalization;
 using System.Text;
 using System.Xml;
 using Maple2.File.IO.Crypto;
@@ -8,14 +7,15 @@
 
 namespace Maple2.File.IO {
     public class M2dReader : IDisposable {
-        private readonly MemoryMappedFile m2dFile;
+        private readonly FileStream m2dFile;
+        private readonly object m2dLock = new();
         public readonly IReadOnlyList<PackFileEntry> Files;
 
         public M2dReader(string path) {
             // Force Globalization to en-US because we use periods instead of commas for decimals
             CultureInfo.CurrentCulture = new("en-US");
 
-            m2dFile = MemoryMappedFile.CreateFromFile(path);
+            m2dFile = System.IO.File.OpenRead(path);
 
             // Create an index from the header file
             using var headerReader = new BinaryReader(System.IO.File.OpenRead(path.Replace(".m2d", ".m2h")));
@@ -41,12 +41,12 @@ public PackFileEntry GetEntry(string filename) {
         }
 
         public XmlReader GetXmlReader(PackFileEntry entry) {
-            return XmlReader.Create(new MemoryStream(CryptoManager.DecryptData(entry.FileHeader, m2dFile)));
+            return XmlReader.Create(new MemoryStream(CryptoManager.DecryptData(entry.FileHeader, m2dFile, m2dLock)));
         }
 
         public XmlDocument GetXmlDocument(PackFileEntry entry) {
             var document = new XmlDocument();
-            byte[] data = CryptoManager.DecryptData(entry.FileHeader, m2dFile);
+            byte[] data = CryptoManager.DecryptData(entry.FileHeader, m2dFile, m2dLock);
             try {
                 document.Load(new MemoryStream(data));
             } catch {
@@ -58,11 +58,11 @@ public XmlDocument GetXmlDocument(PackFileEntry entry) {
         }
 
         public byte[] GetBytes(PackFileEntry entry) {
-            return CryptoManager.DecryptData(entry.FileHeader, m2dFile);
+            return CryptoManager.DecryptData(entry.FileHeader, m2dFile, m2dLock);
         }
 
         public string GetString(PackFileEntry entry) {
-            byte[] data = CryptoManager.DecryptData(entry.FileHeader, m2dFile);
+            byte[] data = CryptoManager.DecryptData(entry.FileHeader, m2dFile, m2dLock);
             string result = Encoding.Default.GetString(data);
             // Remove UTF-8 BOM if present
             if (result.Length > 0 && result[0] == '\uFEFF') {
diff --git a/Maple2.File.Parser/NifParser.cs b/Maple2.File.Parser/NifParser.cs
@@ -17,7 +17,15 @@ public NifParser(List<PrefixedM2dReader> modelM2dReaders) {
                 string path = nifReader.PathPrefix + entry.Name;
                 uint llid = LlidHash.Hash(path);
 
-                NifDocument nifDocument = new NifDocument(path, nifReader.GetBytes(entry));
+                byte[] data;
+                try {
+                    data = nifReader.GetBytes(entry);
+                } catch (Exception ex) {
+                    Console.Error.WriteLine($"[NifParser] Failed to decrypt: {path} from {nifReader.PathPrefix} (FileIndex={entry.FileHeader?.FileIndex}): {ex.Message}");
+                    continue;
+                }
+
+                NifDocument nifDocument = new NifDocument(path, data);
 
                 yield return (llid, path, nifDocument);
             }
diff --git a/Maple2.File.Tests/TestUtils.cs b/Maple2.File.Tests/TestUtils.cs
@@ -23,22 +23,22 @@ static TestUtils() {
             throw new Exception("MS2_DATA_FOLDER is not set");
         }
 
-        XmlReader = new M2dReader(@$"{m2dPath}\Xml.m2d");
-        ExportedReader = new M2dReader(@$"{m2dPath}\Resource\Exported.m2d");
-        ServerReader = new M2dReader(@$"{m2dPath}\Server.m2d");
-        AssetMetadataReader = new M2dReader(@$"{m2dPath}\Resource\asset-web-metadata.m2d");
+        XmlReader = new M2dReader(Path.Combine(m2dPath, "Xml.m2d"));
+        ExportedReader = new M2dReader(Path.Combine(m2dPath, "Resource", "Exported.m2d"));
+        ServerReader = new M2dReader(Path.Combine(m2dPath, "Server.m2d"));
+        AssetMetadataReader = new M2dReader(Path.Combine(m2dPath, "Resource", "asset-web-metadata.m2d"));
         AssetIndex = new AssetIndex(AssetMetadataReader);
         ModelM2dReaders = new List<PrefixedM2dReader>() {
-            new PrefixedM2dReader("/library/", $@"{m2dPath}\Resource\Library.m2d"),
-            new PrefixedM2dReader("/model/map/", $@"{m2dPath}\Resource\Model\Map.m2d"),
-            new PrefixedM2dReader("/model/effect/", $@"{m2dPath}\Resource\Model\Effect.m2d"),
-            new PrefixedM2dReader("/model/camera/", $@"{m2dPath}\Resource\Model\Camera.m2d"),
-            new PrefixedM2dReader("/model/tool/", $@"{m2dPath}\Resource\Model\Tool.m2d"),
-            new PrefixedM2dReader("/model/item/", $@"{m2dPath}\Resource\Model\Item.m2d"),
-            new PrefixedM2dReader("/model/npc/", $@"{m2dPath}\Resource\Model\Npc.m2d"),
-            new PrefixedM2dReader("/model/path/", $@"{m2dPath}\Resource\Model\Path.m2d"),
-            new PrefixedM2dReader("/model/character/", $@"{m2dPath}\Resource\Model\Character.m2d"),
-            new PrefixedM2dReader("/model/textures/", $@"{m2dPath}\Resource\Model\Textures.m2d"),
+            new PrefixedM2dReader("/library/", Path.Combine(m2dPath, "Resource", "Library.m2d")),
+            new PrefixedM2dReader("/model/map/", Path.Combine(m2dPath, "Resource", "Model", "Map.m2d")),
+            new PrefixedM2dReader("/model/effect/", Path.Combine(m2dPath, "Resource", "Model", "Effect.m2d")),
+            new PrefixedM2dReader("/model/camera/", Path.Combine(m2dPath, "Resource", "Model", "Camera.m2d")),
+            new PrefixedM2dReader("/model/tool/", Path.Combine(m2dPath, "Resource", "Model", "Tool.m2d")),
+            new PrefixedM2dReader("/model/item/", Path.Combine(m2dPath, "Resource", "Model", "Item.m2d")),
+            new PrefixedM2dReader("/model/npc/", Path.Combine(m2dPath, "Resource", "Model", "Npc.m2d")),
+            new PrefixedM2dReader("/model/path/", Path.Combine(m2dPath, "Resource", "Model", "Path.m2d")),
+            new PrefixedM2dReader("/model/character/", Path.Combine(m2dPath, "Resource", "Model", "Character.m2d")),
+            new PrefixedM2dReader("/model/textures/", Path.Combine(m2dPath, "Resource", "Model", "Textures.m2d")),
         };
     }
 
