Skip to content

Commit b9c6c95

Browse files
AXiX-officialAXiX-official
committed
Stream Parse
1 parent 41aa3ab commit b9c6c95

34 files changed

Lines changed: 1042 additions & 816 deletions

‎UnityAsset.NET/Asset.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@ namespace UnityAsset.NET;
77
public class Asset
88
{
99
public AssetFileInfo Info;
10-
public DataBuffer RawData;
10+
public IReader RawData;
1111
public NodeData NodeData;
1212

13-
public Asset(AssetFileInfo info, DataBuffer db)
13+
public Asset(AssetFileInfo info, IReader reader)
1414
{
1515
Info = info;
16-
RawData = db;
17-
NodeData = new NodeData(db, info.Type.Nodes, info.Type.Nodes[0]);
16+
RawData = reader;
17+
NodeData = new NodeData(reader, info.Type.Nodes, info.Type.Nodes[0]);
1818
}
1919
}

‎UnityAsset.NET/Enums/Endianness.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
namespace UnityAsset.NET.Enums;
2+
3+
public enum Endianness : byte
4+
{
5+
LittleEndian,
6+
BigEndian
7+
}

‎UnityAsset.NET/Extensions/BundleFileExtensions.cs‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,18 @@ public static void ParseFilesWithTypeConversion(this BundleFile bf)
1515
for (int i = 0; i < filesSpan.Length; i++)
1616
{
1717
ref var file = ref filesSpan[i];
18-
if (file is { File: DataBuffer db, CanBeSerializedFile: true })
19-
file = new FileWrapper(SerializedFile.Parse(db), file.Info);
18+
if (file is { File: IReader reader, CanBeSerializedFile: true })
19+
{
20+
try
21+
{
22+
file = new FileWrapper(SerializedFile.Parse(reader), file.Info);
23+
Console.WriteLine($"File {file.Info.Path} loaded.");
24+
}
25+
catch (Exception e)
26+
{
27+
Console.WriteLine($"Error loading file {file.Info.Path}: {e.Message}");
28+
}
29+
}
2030
}
2131
}
2232

@@ -26,12 +36,12 @@ public static List<Asset> Assets(this BundleFile bf) =>
2636
.SelectMany(file => ((SerializedFile)file.File).Assets)
2737
.ToList();
2838

29-
public static void PatchCrc32(this BundleFile bf, uint newCrc32)
39+
/*public static void PatchCrc32(this BundleFile bf, uint newCrc32)
3040
{
3141
if (bf.Crc32 != newCrc32)
3242
{
3343
var patchBytes = CRC32.rCRC(newCrc32, bf.Crc32);
3444
bf.Files.Add(new FileWrapper(new DataBuffer(patchBytes), new FileEntry(0, 4, 0, "crc32-patch-data")));
3545
}
36-
}
46+
}*/
3747
}

‎UnityAsset.NET/Files/BundleFiles/BlocksAndDirectoryInfo.cs‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,17 @@ public BlocksAndDirectoryInfo(byte[] uncompressedDataHash,
2020
DirectoryInfo = directoryInfo;
2121
}
2222

23-
public static BlocksAndDirectoryInfo Parse(DataBuffer db) => new (
24-
db.ReadBytes(16),
25-
db.ReadList(db.ReadInt32(), StorageBlockInfo.Parse),
26-
db.ReadList(db.ReadInt32(), FileEntry.Parse)
23+
public static BlocksAndDirectoryInfo Parse(IReader reader) => new (
24+
reader.ReadBytes(16),
25+
reader.ReadList(reader.ReadInt32(), StorageBlockInfo.Parse),
26+
reader.ReadList(reader.ReadInt32(), FileEntry.Parse)
2727
);
2828

29-
public int Serialize(DataBuffer db)
29+
public void Serialize(IWriter writer)
3030
{
31-
int size = 0;
32-
size += db.WriteBytes(UncompressedDataHash);
33-
size += db.WriteListWithCount(BlocksInfo, (d, info) => info.Serialize(d));
34-
size += db.WriteListWithCount(DirectoryInfo, (d, info) => info.Serialize(d));
35-
return size;
31+
writer.WriteBytes(UncompressedDataHash);
32+
writer.WriteListWithCount(BlocksInfo, (w, info) => info.Serialize(w));
33+
writer.WriteListWithCount(DirectoryInfo, (w, info) => info.Serialize(w));
3634
}
3735

3836
public long SerializeSize => UncompressedDataHash.Length + 8 +

‎UnityAsset.NET/Files/BundleFiles/BundleFile.cs‎

Lines changed: 63 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using UnityAsset.NET.Files.BundleFiles;
66
using UnityAsset.NET.Files.SerializedFiles;
77
using UnityAsset.NET.IO;
8+
using StreamReader = UnityAsset.NET.IO.StreamReader;
89

910
namespace UnityAsset.NET.Files;
1011

@@ -33,7 +34,7 @@ public class BundleFile
3334

3435
public uint Crc32;
3536

36-
public static UnityCN? ParseUnityCnInfo(DataBuffer db, Header header, string? key)
37+
public static UnityCN? ParseUnityCnInfo(IReader reader, Header header, string? key)
3738
{
3839
var unityCnMask = VersionJudge1(ParseVersion(header))
3940
? ArchiveFlags.BlockInfoNeedPaddingAtStart
@@ -44,45 +45,45 @@ public class BundleFile
4445
key ??= Setting.DefaultUnityCNKey;
4546
if (key == null)
4647
throw new Exception($"UnityCN key is required for decryption. UnityCN Flag Mask: {unityCnMask}");
47-
return new UnityCN(db, key);
48+
return new UnityCN(reader, key);
4849
}
4950
return null;
5051
}
5152

52-
public static void AlignAfterHeader(DataBuffer db, Header header)
53+
public static void AlignAfterHeader(IReader reader, Header header)
5354
{
5455
if (header.Version >= 7)
55-
db.Align(16);
56+
reader.Align(16);
5657
else // temp fix for 2019.4.x
5758
{
5859
var version = ParseVersion(header);
5960
if (version[0] == 2019 && version[1] == 4 && version[2] >= 30)
60-
db.Align(16);
61+
reader.Align(16);
6162
}
6263
}
6364

64-
public static BlocksAndDirectoryInfo ParseDataInfo(DataBuffer db, Header header)
65+
public static BlocksAndDirectoryInfo ParseDataInfo(IReader reader, Header header)
6566
{
66-
Span<byte> blocksInfoBytes = (header.Flags & ArchiveFlags.BlocksInfoAtTheEnd) == 0
67-
? db.ReadSpanBytes((int)header.CompressedBlocksInfoSize)
68-
: db[(int)(header.Size - header.CompressedBlocksInfoSize)..(int)header.Size];
67+
var blocksInfoBytes = (header.Flags & ArchiveFlags.BlocksInfoAtTheEnd) == 0
68+
? reader.ReadOnlySpanBytes((int)header.CompressedBlocksInfoSize)
69+
: reader.ReadOnlySlice((int)(header.Size - header.CompressedBlocksInfoSize), (int)header.CompressedBlocksInfoSize);
6970
var compressionType = (CompressionType)(header.Flags & ArchiveFlags.CompressionTypeMask);
70-
DataBuffer blocksInfoUncompressedData = new DataBuffer((int)header.UncompressedBlocksInfoSize);
71-
Compression.DecompressToBytes(blocksInfoBytes, blocksInfoUncompressedData.AsSpan(), compressionType);
71+
MemoryBinaryIO blocksInfoUncompressedData = MemoryBinaryIO.Create((int)header.UncompressedBlocksInfoSize);
72+
Compression.DecompressToBytes(blocksInfoBytes, blocksInfoUncompressedData.AsWritableSpan, compressionType);
7273
var dataInfo = BlocksAndDirectoryInfo.Parse(blocksInfoUncompressedData);
7374
if (!VersionJudge1(ParseVersion(header)) && (header.Flags & ArchiveFlags.BlockInfoNeedPaddingAtStart) != 0)
74-
db.Align(16);
75+
reader.Align(16);
7576
return dataInfo;
7677
}
7778

78-
public static (List<FileWrapper>, uint) ParseFiles(DataBuffer db, BlocksAndDirectoryInfo dataInfo, UnityCN? unityCnInfo = null)
79+
public static (List<FileWrapper>, uint) ParseFiles(IReader reader, BlocksAndDirectoryInfo dataInfo, UnityCN? unityCnInfo = null)
7980
{
80-
DataBuffer blocksBuffer = new DataBuffer(dataInfo.BlocksInfo.Sum(block => (int)block.UncompressedSize));
81+
MemoryBinaryIO blocksBuffer = MemoryBinaryIO.Create(dataInfo.BlocksInfo.Sum(block => (int)block.UncompressedSize));
8182
if (unityCnInfo == null)
8283
foreach (var blockInfo in dataInfo.BlocksInfo.AsReadOnlySpan())
8384
Compression.DecompressToBytes(
84-
db.ReadSpanBytes((int)blockInfo.CompressedSize),
85-
blocksBuffer.ReadSpanBytes((int)blockInfo.UncompressedSize),
85+
reader.ReadOnlySpanBytes((int)blockInfo.CompressedSize),
86+
blocksBuffer.GetWritableSpan((int)blockInfo.UncompressedSize),
8687
(CompressionType)(blockInfo.Flags & StorageBlockFlags.CompressionTypeMask));
8788
else
8889
{
@@ -93,23 +94,35 @@ public static (List<FileWrapper>, uint) ParseFiles(DataBuffer db, BlocksAndDirec
9394
var compressionType = (CompressionType)(blockInfo.Flags & StorageBlockFlags.CompressionTypeMask);
9495
if (compressionType == CompressionType.Lz4 || compressionType == CompressionType.Lz4HC)
9596
unityCnInfo.DecryptAndDecompress(
96-
db.ReadSpanBytes((int)blockInfo.CompressedSize),
97-
blocksBuffer.ReadSpanBytes((int)blockInfo.UncompressedSize),
97+
reader.ReadOnlySpanBytes((int)blockInfo.CompressedSize),
98+
blocksBuffer.GetWritableSpan((int)blockInfo.UncompressedSize),
9899
i);
99100
else
100101
throw new IOException($"Unsupported compression type {compressionType} for UnityCN");
101102
}
102103
}
103104
blocksBuffer.Seek(0);
104105

105-
var crc32 = CRC32.CalculateCRC32(blocksBuffer.AsSpan());
106+
var crc32 = CRC32.CalculateCRC32(blocksBuffer.AsReadOnlySpan);
106107

107108
var files = new List<FileWrapper>();
108109
foreach (var dir in dataInfo.DirectoryInfo.AsReadOnlySpan())
109-
files.Add(new FileWrapper(new DataBuffer(blocksBuffer.ReadSpanBytes((int)dir.Size).ToArray()), dir));
110+
files.Add(new FileWrapper(MemoryBinaryIO.Create(blocksBuffer.ReadOnlySpanBytes((int)dir.Size).ToArray()), dir));
110111

111112
return (files, crc32);
112113
}
114+
115+
// crc calculate disabled
116+
public static (List<FileWrapper>, uint) LazyParseFiles(StreamReader reader, BlocksAndDirectoryInfo dataInfo, UnityCN? unityCnInfo = null)
117+
{
118+
var blockStream = new BlockStream(dataInfo.BlocksInfo, reader, unityCnInfo);
119+
//var crc32 = CRC32.CalculateCRC32(blocksBuffer.AsReadOnlySpan);
120+
var files = new List<FileWrapper>();
121+
foreach (var dir in dataInfo.DirectoryInfo.AsReadOnlySpan())
122+
files.Add(new FileWrapper(new FileEntryStreamReader(blockStream, dir), dir));
123+
124+
return (files, 0);
125+
}
113126

114127
public BundleFile(Header header, BlocksAndDirectoryInfo dataInfo, List<FileWrapper> files, string? key = null)
115128
{
@@ -119,14 +132,14 @@ public BundleFile(Header header, BlocksAndDirectoryInfo dataInfo, List<FileWrapp
119132
Files = files;
120133
}
121134

122-
public BundleFile(DataBuffer db, string? key = null)
135+
public BundleFile(IReader reader, string? key = null)
123136
{
124137
UnityCnKey = key;
125-
Header = Header.Parse(db);
126-
UnityCnInfo = ParseUnityCnInfo(db, Header, UnityCnKey);
127-
AlignAfterHeader(db, Header);
128-
DataInfo = ParseDataInfo(db, Header);
129-
(Files, Crc32) = ParseFiles(db, DataInfo, UnityCnInfo);
138+
Header = Header.Parse(reader);
139+
UnityCnInfo = ParseUnityCnInfo(reader, Header, UnityCnKey);
140+
AlignAfterHeader(reader, Header);
141+
DataInfo = ParseDataInfo(reader, Header);
142+
(Files, Crc32) = (reader is StreamReader streamReader) ? LazyParseFiles(streamReader, DataInfo, UnityCnInfo) : ParseFiles(reader, DataInfo, UnityCnInfo);
130143
}
131144

132145
/// <summary>
@@ -138,28 +151,28 @@ public BundleFile(DataBuffer db, string? key = null)
138151
/// </remarks>
139152
/// <param name="path">The file path to the BundleFile</param>
140153
/// <param name="key">Optional decryption key</param>
141-
public BundleFile(string path, string? key = null) : this(DataBuffer.FromFile(path), key) {}
154+
public BundleFile(string path, string? key = null) : this(new FileStreamReader(path), key) {}
142155

143-
public int Serialize(DataBuffer db, CompressionType infoPacker = CompressionType.None, CompressionType dataPacker = CompressionType.None, string? unityCnKey = null)
156+
/*public void Serialize(IWriter writer, CompressionType infoPacker = CompressionType.None, CompressionType dataPacker = CompressionType.None, string? unityCnKey = null)
144157
{
145158
if (Header == null || DataInfo == null || Files == null)
146159
throw new NullReferenceException("BundleFile has not read completely");
147160
var directoryInfo = new List<FileEntry>();
148161
var filesCapacity = Files.Sum(c =>
149162
{
150-
if (c.File is DataBuffer hdb) return hdb.Capacity;
163+
if (c.File is IReader r) return r.Length;
151164
if (c.File is SerializedFile sf) return sf.SerializeSize;
152165
return 0;
153166
});
154-
DataBuffer filesBuffer = new DataBuffer((int)filesCapacity);
167+
MemoryBinaryIO filesBuffer = MemoryBinaryIO.Create((int)filesCapacity);
155168
Int64 offset = 0;
156169
foreach (var file in Files.AsReadOnlySpan())
157170
{
158171
int cabSize;
159172
switch (file.File)
160173
{
161-
case DataBuffer dataBuffer:
162-
cabSize = filesBuffer.WriteBytes(dataBuffer.AsSpan());
174+
case IReader r:
175+
cabSize = filesBuffer.WriteBytes(r.);
163176
break;
164177
case SerializedFile sf:
165178
cabSize = sf.Serialize(filesBuffer.SliceBufferToEnd());
@@ -182,14 +195,14 @@ public int Serialize(DataBuffer db, CompressionType infoPacker = CompressionType
182195
int chunkSize = (int)Math.Min(blocksSize, defaultChunkSize);
183196
var compressedSize = Compression.CompressToBytes(
184197
filesBuffer.ReadSpanBytes(chunkSize),
185-
compressedBlocksDataBuffer.SliceForward(),
198+
compressedBlocksDataBuffer.SliceForward(),
186199
dataPacker);
187200
blocksInfo.Add(new StorageBlockInfo((UInt32)chunkSize, (UInt32)compressedSize, (StorageBlockFlags)dataPacker));
188201
blocksSize -= chunkSize;
189202
compressedBlocksDataBuffer.Advance((int)compressedSize);
190203
}
191204
compressedBlocksDataBuffer.Seek(0);
192-
205+
193206
UnityCN? unityCnInfo = null;
194207
var version = ParseVersion(Header);
195208
if (unityCnKey != null)
@@ -212,14 +225,14 @@ public int Serialize(DataBuffer db, CompressionType infoPacker = CompressionType
212225
else if (UnityCnKey != null)
213226
{
214227
var unityCnMask = VersionJudge1(version)
215-
? ArchiveFlags.BlockInfoNeedPaddingAtStart
228+
? ArchiveFlags.BlockInfoNeedPaddingAtStart
216229
: ArchiveFlags.UnityCNEncryption | ArchiveFlags.UnityCNEncryptionNew;
217230
if (unityCnMask == (ArchiveFlags.UnityCNEncryption | ArchiveFlags.UnityCNEncryptionNew))
218231
unityCnMask = (Header.Flags & ArchiveFlags.UnityCNEncryptionNew) != 0 ?
219232
ArchiveFlags.UnityCNEncryptionNew : ArchiveFlags.UnityCNEncryption;
220233
Header.Flags &= ~unityCnMask;
221234
}
222-
235+
223236
var dataInfo = new BlocksAndDirectoryInfo(DataInfo.UncompressedDataHash, blocksInfo, directoryInfo);
224237
DataBuffer dataInfoBuffer = new DataBuffer((int)dataInfo.SerializeSize);
225238
dataInfo.Serialize(dataInfoBuffer);
@@ -230,37 +243,37 @@ public int Serialize(DataBuffer db, CompressionType infoPacker = CompressionType
230243
var header = new Header(Header.Signature, Header.Version, Header.UnityVersion, Header.UnityRevision,
231244
0, (uint)compressedBlocksInfoSize, (uint)uncompressedBlocksInfoSize,
232245
(Header.Flags & ~ArchiveFlags.CompressionTypeMask) | (ArchiveFlags)infoPacker);
233-
db.EnsureCapacity((int)(header.SerializeSize + 16 + compressedBlocksInfoSize + compressedBlocksDataBuffer.Length + unityCnInfo?.SerializeSize ?? 0));
246+
writer.EnsureCapacity((int)(header.SerializeSize + 16 + compressedBlocksInfoSize + compressedBlocksDataBuffer.Length + unityCnInfo?.SerializeSize ?? 0));
234247
int size = 0;
235-
size += header.Serialize(db);
248+
size += header.Serialize(writer);
236249
if (unityCnInfo != null)
237-
size += unityCnInfo.Serialize(db);
250+
size += unityCnInfo.Serialize(writer);
238251
if (header.Version >= 7)
239-
size += db.Align(16);
252+
size += writer.Align(16);
240253
else // temp fix for 2019.4.x
241254
if (version[0] == 2019 && version[1] == 4 && version[2] >= 30)
242-
size += db.Align(16);
255+
size += writer.Align(16);
243256
if ((header.Flags & ArchiveFlags.BlocksInfoAtTheEnd) == 0) //0x40 BlocksAndDirectoryInfoCombined
244-
size += db.WriteBytes(compressedDataInfoBuffer.SliceForward((int)compressedBlocksInfoSize));
257+
size += writer.WriteBytes(compressedDataInfoBuffer.SliceForward((int)compressedBlocksInfoSize));
245258
if (!VersionJudge1(version) &&(header.Flags & ArchiveFlags.BlockInfoNeedPaddingAtStart) != 0)
246-
size += db.Align(16);
247-
size += db.WriteBytes(compressedBlocksDataBuffer.SliceForward(compressedBlocksDataBuffer.Length));
259+
size += writer.Align(16);
260+
size += writer.WriteBytes(compressedBlocksDataBuffer.SliceForward(compressedBlocksDataBuffer.Length));
248261
if ((header.Flags & ArchiveFlags.BlocksInfoAtTheEnd) != 0) //kArchiveBlocksInfoAtTheEnd
249-
size += db.WriteBytes(compressedDataInfoBuffer.SliceForward((int)compressedBlocksInfoSize));
262+
size += writer.WriteBytes(compressedDataInfoBuffer.SliceForward((int)compressedBlocksInfoSize));
250263
header.Size = size;
251-
db.Seek(0);
252-
header.Serialize(db);
264+
writer.Seek(0);
265+
header.Serialize(writer);
253266
return size;
254-
}
267+
}*/
255268

256-
public int Serialize(string path, CompressionType infoPacker = CompressionType.None,
269+
/*public int Serialize(string path, CompressionType infoPacker = CompressionType.None,
257270
CompressionType dataPacker = CompressionType.None, string? unityCnKey = null)
258271
{
259272
DataBuffer db = new DataBuffer(0);
260273
int size = Serialize(db, infoPacker, dataPacker, unityCnKey);
261274
db.WriteToFile(path, size);
262275
return size;
263-
}
276+
}*/
264277

265278
public override string ToString()
266279
{

‎UnityAsset.NET/Files/BundleFiles/FileEntry.cs‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,21 +18,19 @@ public FileEntry(Int64 offset, Int64 size, UInt32 flags, string path)
1818
Path = path;
1919
}
2020

21-
public static FileEntry Parse(DataBuffer db) => new (
22-
db.ReadInt64(),
23-
db.ReadInt64(),
24-
db.ReadUInt32(),
25-
db.ReadNullTerminatedString()
21+
public static FileEntry Parse(IReader reader) => new (
22+
reader.ReadInt64(),
23+
reader.ReadInt64(),
24+
reader.ReadUInt32(),
25+
reader.ReadNullTerminatedString()
2626
);
2727

28-
public int Serialize(DataBuffer db)
28+
public void Serialize(IWriter writer)
2929
{
30-
int size = 0;
31-
size += db.WriteInt64(Offset);
32-
size += db.WriteInt64(Size);
33-
size += db.WriteUInt32(Flags);
34-
size += db.WriteNullTerminatedString(Path);
35-
return size;
30+
writer.WriteInt64(Offset);
31+
writer.WriteInt64(Size);
32+
writer.WriteUInt32(Flags);
33+
writer.WriteNullTerminatedString(Path);
3634
}
3735

3836
public long SerializeSize => 21 + Path.Length;

0 commit comments

Comments
 (0)