Boilerplate, Refactors, Resource Hygiene

Volatility/CLI/Commands/ExportResourceCommand.cs

@@ -10,11 +10,13 @@ internal class ExportResourceCommand : ICommand
 {
         public static string CommandToken => "ExportResource";
         public static string CommandDescription => "Exports information and relevant data from an imported/created resource into a platform's format.";
-        public static string CommandParameters => "--recurse --overwrite --type=<resource type OR index> --format=<tub,bpr,x360,ps3> --respath=<data path> --outpath=<file path>";
+        public static string CommandParameters => "--recurse --overwrite --type=<resource type OR index> --format=<tub,bpr,x360,ps3> --respath=<data path> --outpath=<file path> [--imports=<raw,bnd2manager,dgi,yap,volatility>] [--importsfile]";
 
         public string? Format { get; set; }
         public string? ResourcePath { get; set; }
         public string? OutputPath { get; set; }
+        public string? Imports { get; set; }
+        public bool ImportsFile { get; set; }
         public bool Overwrite { get; set; }
         public bool Recursive { get; set; }
 
@@ -57,6 +59,18 @@ public async Task Execute()
                     throw new InvalidPlatformException("Error: Invalid file format specified!");
                 }
 
+                Unpacker? importUnpackerOverride = null;
+                if (!string.IsNullOrEmpty(Imports) && !string.Equals(Imports, "DEFAULT", StringComparison.OrdinalIgnoreCase))
+                {
+                    if (!TypeUtilities.TryParseEnum(Imports, out Unpacker parsedUnpacker))
+                    {
+                        Console.WriteLine("Error: Invalid imports export mode specified!");
+                        return;
+                    }
+
+                    importUnpackerOverride = parsedUnpacker;
+                }
+
                 var loadOperation = new LoadResourceOperation();
                 var exportOperation = new ExportResourceOperation();
 
@@ -104,7 +118,7 @@ public async Task Execute()
                     return;
                 }
 
-                                await exportOperation.ExecuteAsync(resource, OutputPath, platform);
+                                await exportOperation.ExecuteAsync(resource, OutputPath, platform, importUnpackerOverride, ImportsFile);
 
                 Console.WriteLine($"Exported {Path.GetFileName(ResourcePath)} as {Path.GetFullPath(OutputPath)}.");
                         }));
@@ -117,6 +131,8 @@ public void SetArgs(Dictionary<string, object> args)
                 Format = (args.TryGetValue("format", out object? format) ? format as string : "")?.ToUpper();
                 ResourcePath = args.TryGetValue("respath", out object? respath) ? respath as string : "";
                 OutputPath = args.TryGetValue("outpath", out object? outpath) ? outpath as string : "";
+                Imports = args.TryGetValue("imports", out object? imports) ? imports as string : "";
+                ImportsFile = args.TryGetValue("importsfile", out var importsfile) && (bool)importsfile;
                 Overwrite = args.TryGetValue("overwrite", out var ow) && (bool)ow;
                 Recursive = args.TryGetValue("recurse", out var re) && (bool)re;
         }

Volatility/Frontend.cs

@@ -38,8 +38,7 @@ static void Main(string[] args)
     {
         if (args.Length > 0)
         {
-            string fullCommand = string.Join(" ", args);
-            RunCommand(fullCommand);
+            RunCommandTokenized(args);
         }
         else 
         {
@@ -128,6 +127,31 @@ static void RunCommand(string input)
         }
     }
 
+    static void RunCommandTokenized(string[] input)
+    {
+        if (input.Length == 0)
+        {
+            return;
+        }
+
+        try
+        {
+            var command = ParseCommandTokenized(input);
+            command.Execute().GetAwaiter().GetResult();
+
+            GC.Collect();
+            GC.WaitForPendingFinalizers();
+            GC.Collect();
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine($"Error: {ex.Message}");
+#if DEBUG
+            throw;
+#endif
+        }
+    }
+
     static ICommand ParseCommandTokenized(string[] input)    // Split command
     {
         var commandName = input[0].ToLower();

Volatility/Operations/Resources/ExportResourceOperation.cs

@@ -1,11 +1,15 @@
 using Volatility.Resources;
 using Volatility.Utilities;
-
 namespace Volatility.Operations.Resources;
 
 internal class ExportResourceOperation
 {
-    public Task ExecuteAsync(Resource resource, string outputPath, Platform platform)
+    public Task ExecuteAsync(
+        Resource resource,
+        string outputPath,
+        Platform platform,
+        Unpacker? importUnpackerOverride = null,
+        bool writeImportsToSeparateFile = false)
     {
         string? directoryPath = Path.GetDirectoryName(outputPath);
 
@@ -32,6 +36,14 @@ public Task ExecuteAsync(Resource resource, string outputPath, Platform platform
                 break;
         }
 
+        WriteExternalImports(
+            resource,
+            outputPath,
+            writer,
+            endian,
+            ResolveExternalImportsUnpackerFormat(resource, importUnpackerOverride),
+            writeImportsToSeparateFile);
+
         if (resource is ShaderBase shader)
         {
             var stages = shader.GetCompileStages();
@@ -70,6 +82,94 @@ public Task ExecuteAsync(Resource resource, string outputPath, Platform platform
         return Task.CompletedTask;
     }
 
+    private static Unpacker ResolveExternalImportsUnpackerFormat(
+        Resource resource,
+        Unpacker? importUnpackerOverride)
+    {
+        return importUnpackerOverride ?? resource.Unpacker;
+    }
+
+    private static void WriteExternalImports(
+        Resource resource,
+        string outputPath,
+        ResourceBinaryWriter writer,
+        Endian endian,
+        Unpacker importUnpacker,
+        bool forceExternalImportsFile)
+    {
+        List<KeyValuePair<long, ResourceImport>> imports = resource.GetExternalImports().ToList();
+
+        string yamlImportsPath = ResourceImport.GetImportsPath(outputPath, Unpacker.YAP);
+        string datImportsPath = ResourceImport.GetImportsPath(outputPath, Unpacker.Raw);
+
+        if (imports.Count == 0)
+        {
+            ResourceImport.DeleteImportsSidecarFiles(outputPath);
+            return;
+        }
+
+        if (importUnpacker == Unpacker.YAP)
+        {
+            ResourceImport.DeleteImportsSidecarFiles(outputPath);
+            WriteExternalImportsYaml(imports, yamlImportsPath);
+            return;
+        }
+
+        if (forceExternalImportsFile)
+        {
+            ResourceImport.DeleteImportsSidecarFiles(outputPath);
+            WriteExternalImportsDat(datImportsPath, endian, imports);
+            return;
+        }
+
+        writer.BaseStream.Seek(0, SeekOrigin.End);
+        WriteBinaryImports(writer, imports);
+        ResourceImport.DeleteImportsSidecarFiles(outputPath);
+    }
+
+    private static void WriteExternalImportsYaml(
+        List<KeyValuePair<long, ResourceImport>> imports,
+        string importsPath)
+    {
+        List<string> lines = new(imports.Count);
+        foreach (KeyValuePair<long, ResourceImport> entry in imports)
+        {
+            ulong resourceId = ResourceUtilities.ResolveResourceID(entry.Value);
+            lines.Add($"- \"0x{entry.Key:x8}\": \"{resourceId:X8}\"");
+        }
+
+        File.WriteAllLines(importsPath, lines);
+    }
+
+    private static void WriteExternalImportsDat(
+        string importsPath,
+        Endian endianness,
+        List<KeyValuePair<long, ResourceImport>> imports)
+    {
+        using FileStream fs = new(importsPath, FileMode.Create, FileAccess.Write);
+        using EndianAwareBinaryWriter writer = new(fs, endianness);
+        WriteBinaryImports(writer, imports);
+    }
+
+    private static void WriteBinaryImports(
+        EndianAwareBinaryWriter writer,
+        List<KeyValuePair<long, ResourceImport>> imports)
+    {
+        foreach (KeyValuePair<long, ResourceImport> entry in imports)
+        {
+            if (entry.Key < 0 || entry.Key > uint.MaxValue)
+            {
+                throw new InvalidDataException(
+                    $"Import offset 0x{entry.Key:X} cannot be stored in a binary imports block.");
+            }
+
+            // Probably overkill but I just want to make sure we always use the correct writer overloads
+            writer.Write((ulong)ResourceUtilities.ResolveResourceID(entry.Value));
+            writer.Write((uint)entry.Key);
+            writer.Write(0x00000000);
+        }
+    }
+
     private static string GetShaderProgramBufferPath(
         string shaderOutputPath,
         ShaderStageCompile stage,

Volatility/Operations/Resources/ImportResourceOperation.cs

@@ -1,4 +1,3 @@
-using System.Diagnostics;
 using System.Text.RegularExpressions;
 
 using Volatility.Resources;
@@ -105,33 +104,10 @@ public async Task<ImportResourceResult> ExecuteAsync(ResourceType resourceType,
 
                         if (!File.Exists(convertedSamplePathName) || overwrite)
                         {
-                            ProcessStartInfo start = new ProcessStartInfo
-                            {
-                                FileName = sxPath,
-                                Arguments = $"-wave -s16l_int -v0 \"{samplePathName}.snr\" -=\"{convertedSamplePathName}\"",
-                                RedirectStandardOutput = true,
-                                RedirectStandardError = true,
-                                UseShellExecute = false,
-                                CreateNoWindow = true
-                            };
-
-                            using Process process = new Process();
-                            process.StartInfo = start;
-                            process.OutputDataReceived += (sender, e) =>
-                            {
-                                if (!string.IsNullOrEmpty(e.Data)) Console.WriteLine(e.Data);
-                            };
-
-                            process.ErrorDataReceived += (sender, e) =>
-                            {
-                                if (!string.IsNullOrEmpty(e.Data)) Console.WriteLine(e.Data);
-                            };
-
                             Console.WriteLine($"Converting extracted sample {sampleName}.snr to wave...");
-                            process.Start();
-                            process.BeginOutputReadLine();
-                            process.BeginErrorReadLine();
-                            process.WaitForExit();
+                            ProcessUtilities.RunAndRelayOutput(
+                                sxPath,
+                                $"-wave -s16l_int -v0 \"{samplePathName}.snr\" -=\"{convertedSamplePathName}\"");
                         }
                         else
                         {

Volatility/Operations/Resources/SaveResourceOperation.cs

@@ -15,6 +15,7 @@ public SaveResourceOperation()
             .DisableAliases()
             .WithTypeInspector(inner => new IncludeFieldsTypeInspector(inner))
             .WithTypeConverter(new ResourceYamlTypeConverter())
+            .WithTypeConverter(new BitArrayYamlTypeConverter())
             .WithTypeConverter(new StrongIDYamlTypeConverter())
             .WithTypeConverter(new StringEnumYamlTypeConverter())
             .Build();

Volatility/Resources/AptData/AptData.cs

@@ -2,11 +2,10 @@
 
 namespace Volatility.Resources;
 
+[ResourceDefinition(ResourceType.AptData)]
+[ResourceRegistration(RegistrationPlatforms.All, EndianMapped = true)]
 public class AptData : Resource
 {
-    public override ResourceType ResourceType => ResourceType.AptData;
-    public override Platform ResourcePlatform => Platform.Agnostic;
-
     public string MovieName;
     public string BaseComponentName;
     public GuiGeometryObject GuiGeometry;
@@ -146,4 +145,4 @@ public enum ETextureType : int
     Vector = 0,
     TexturedClamp = 1,
     TexturedWrap = 2
-}
+}

Volatility/Resources/AttribSysVault/AttribSysVault.cs

@@ -9,11 +9,10 @@ namespace Volatility.Resources;
 // Learn More:
 // https://burnout.wiki/wiki/AttribSysVault
 
+[ResourceDefinition(ResourceType.AttribSysVault)]
+[ResourceRegistration(RegistrationPlatforms.All, EndianMapped = true)]
 public class AttribSysVault : Resource
 {
-    public override ResourceType ResourceType => ResourceType.AttribSysVault;
-    public override Platform ResourcePlatform => Platform.Agnostic;
-
     public ulong VltDataOffset { get; set; }
     public uint VltSizeInBytes { get; set; }
     public ulong BinDataOffset { get; set; }
@@ -76,8 +75,7 @@ public override void ParseFromStream(ResourceBinaryReader reader, Endian endiann
         PtrN = [];
         Data = string.Empty;
 
-        Arch arch = ResourceArch;
-        if (arch == Arch.x64)
+        if (ResourceArch == Arch.x64)
         {
             VltDataOffset = reader.ReadUInt64();
             VltSizeInBytes = reader.ReadUInt32();
@@ -123,7 +121,8 @@ public override void WriteToStream(ResourceBinaryWriter writer, Endian endiannes
 
     public AttribSysVault() : base() { }
 
-    public AttribSysVault(string path, Endian endianness = Endian.Agnostic) : base(path, endianness) { }
+    public AttribSysVault(string path, Endian endianness = Endian.Agnostic)
+        : base(path, endianness) { }
 
     private void ParseVlt(EndianAwareBinaryReader reader, List<PendingAttribute> pendingAttributes)
     {

Volatility/Resources/BinaryResource.cs

@@ -7,22 +7,31 @@ namespace Volatility.Resources;
 // Learn More:
 // https://burnout.wiki/wiki/Binary_File
 
+[ResourceDefinition(ResourceType.BinaryFile)]
+[ResourceRegistration(RegistrationPlatforms.All, EndianMapped = true)]
 public class BinaryResource : Resource
 {
-    public override ResourceType ResourceType => ResourceType.BinaryFile;
-
     public uint DataSize { get; set; }
     public uint DataOffset { get; set; }
 
-    public BinaryResource(uint dataOffset, uint dataSize)
+    public BinaryResource(uint dataOffset, uint dataSize) : this()
     {
         DataSize = dataSize;
-        DataOffset = dataOffset;
+        DataOffset = dataOffset == 0 ? 0x10u : dataOffset;
     }
 
-    public BinaryResource() : base() { }
-
-    public BinaryResource(string path, Endian endianness = Endian.Agnostic) : base(path, endianness) { }
+    public BinaryResource() : base()
+    {
+        DataOffset = 0x10;
+    }
+
+    public BinaryResource(string path, Endian endianness = Endian.Agnostic) : base(path, endianness)
+    {
+        if (DataOffset == 0)
+        {
+            DataOffset = 0x10;
+        }
+    }
 
     public override void ParseFromStream(ResourceBinaryReader reader, Endian endianness = Endian.Agnostic)
     {
@@ -36,8 +45,16 @@ public override void ParseFromStream(ResourceBinaryReader reader, Endian endiann
 
     public override void WriteToStream(ResourceBinaryWriter writer, Endian endianness = Endian.Agnostic)
     {
+        base.WriteToStream(writer, endianness);
+
+        if (DataOffset < 0x10)
+        {
+            DataOffset = 0x10;
+        }
+
         writer.Write(DataSize);
         writer.Write(DataOffset);
         writer.Write(new byte[8]);
+        writer.BaseStream.Seek(DataOffset, SeekOrigin.Begin);
     }
-}
+}