static void HandleEncode(string input, string output, bool legacy, IConsole console)
{
var inputBytes = File.ReadAllBytes(input);
byte[] outputBytes = null;
if (legacy)
{
// Legacy encode.
var amount = MzxYaz.Encode(inputBytes, inputBytes.Length, out var result);
if (amount >= 0)
{
outputBytes = new byte[amount];
Buffer.BlockCopy(result, 0, outputBytes, 0, outputBytes.Length);
}
}
else
{
// New encode.
var amount = Yaz.Encode(inputBytes, out var compressed);
outputBytes = new byte[amount];
compressed.AsSpan(0, amount).CopyTo(outputBytes);
}
File.WriteAllBytes(output, outputBytes);
}
public static byte[] BuildROM()
{
// yaz0 encode all of the files for the rom
Parallel.ForEach(RomData.MMFileList, file =>
{
if (file.IsCompressed && file.WasEdited)
{
// lower priority so that the rando can't lock a badly scheduled CPU by using 100%
var previousThreadPriority = Thread.CurrentThread.Priority;
Thread.CurrentThread.Priority = ThreadPriority.Lowest;
byte[] result;
var newSize = Yaz.Encode(file.Data, file.Data.Length, out result);
if (newSize >= 0)
{
file.Data = new byte[newSize];
ReadWriteUtils.Arr_Insert(result, 0, newSize, file.Data, 0);
}
// this thread is borrowed, we don't want it to always be the lowest priority, return to previous state
Thread.CurrentThread.Priority = previousThreadPriority;
}
});
byte[] ROM = new byte[0x2000000];
int ROMAddr = 0;
// write all files to rom
for (int i = 0; i < RomData.MMFileList.Count; i++)
{
if (RomData.MMFileList[i].Cmp_Addr == -1)
{
continue;
}
RomData.MMFileList[i].Cmp_Addr = ROMAddr;
int fileLength = RomData.MMFileList[i].Data.Length;
if (RomData.MMFileList[i].IsCompressed)
{
RomData.MMFileList[i].Cmp_End = ROMAddr + fileLength;
}
if (ROMAddr + fileLength > ROM.Length) // rom too small
{
// assuming the largest file isn't the last one, we still want some extra space for further files
// padding will reduce the requirements for further resizes
int expansionIncrementSize = 0x40000; // 1mb might be too large, not sure if there is a hardware compatiblity issue here
int expansionLength = (((ROMAddr + fileLength - ROM.Length) / expansionIncrementSize) + 1) * expansionIncrementSize;
byte[] newROM = new byte[ROM.Length + expansionLength];
Buffer.BlockCopy(ROM, 0, newROM, 0, ROM.Length);
Buffer.BlockCopy(new byte[expansionLength], 0, newROM, ROM.Length, expansionLength);
ROM = newROM;
Debug.WriteLine("*** Expanding rom to size 0x" + ROM.Length.ToString("X2") + "***");
}
ReadWriteUtils.Arr_Insert(RomData.MMFileList[i].Data, 0, fileLength, ROM, ROMAddr);
ROMAddr += fileLength;
}
SequenceUtils.UpdateBankInstrumentPointers(ROM);
UpdateFileTable(ROM);
SignROM(ROM);
FixCRC(ROM);
return(ROM);
}
private static void CompressRom_new(Stream rom, DmaData dmadata, List <int> exclusions, Stream sw)
{
BinaryReader br = new BinaryReader(rom);
MemoryStream outRom = new MemoryStream(0x200_0000);
List <FileRecord> newDmaTable = new List <FileRecord>();
Console.WriteLine();
int cur = 0;
for (int i = 0; i < dmadata.Table.Count; i++)
{
FileRecord record = dmadata.Table[i];
if (record.VRom.End == 0)
{
var r = new FileRecord(new FileAddress(), new FileAddress());
newDmaTable.Add(r);
break;
}
br.BaseStream.Position = record.Rom.Start;
byte[] data = br.ReadBytes(record.VRom.Size);
int dstsize;
FileAddress physical;
if (!exclusions.Contains(i))
{
dstsize = Yaz.Encode(data, data.Length, outRom);
dstsize = Align.To16(dstsize);
physical = new FileAddress(cur, cur + dstsize);
}
else
{
dstsize = data.Length;
dstsize = Align.To16(dstsize);
outRom.Write(data, 0, data.Length);
physical = new FileAddress(cur, 0);
exclusions.Remove(i);
}
var newRec = new FileRecord(record.VRom, physical);
newDmaTable.Add(newRec);
cur += dstsize;
outRom.Position = cur;
}
br.Close();
WriteFileTable(outRom, dmadata.Address.VRom, newDmaTable);
CRC.Write(outRom);
outRom.Position = 0;
outRom.CopyTo(sw);
}
public byte[] EncodeNew()
{
byte[] result = null;
foreach (var sample in Samples)
{
result = Yaz.Decode(sample.AsSpan());
var length = Yaz.Encode(result, out var encoded);
result = encoded;
}
return(result);
}
private void CompressFile(List <string> file)
{
ORom r = new ORom(file[0], ORom.Build.MQP);
byte[] data;
using (Stream stream = r.Files.GetFile(COMPRESS_FILE_TEST))
using (FileStream fs = new FileStream("00E7DC20", FileMode.CreateNew))
{
data = new byte[stream.Length];
stream.Read(data, 0, (int)stream.Length);
Yaz.Encode(data, data.Length, fs);
}
}
public static void TestYazCompression(IExperimentFace face, List <string> filePath)
{
ORom rom = new ORom(filePath[0], ORom.Build.N0);
StringBuilder sb = new StringBuilder();
foreach (var file in rom.Files)
{
if (!file.IsCompressed)
{
continue;
}
try
{
Stream vanillaFile = rom.Files.GetPhysicalFile(file.VRom);
var decompressedFile = Yaz.Decode(vanillaFile, file.Rom.Size);
MemoryStream ms = new MemoryStream(file.Rom.Size);
sb.AppendLine($"{ Yaz.Encode(decompressedFile, decompressedFile.Length, ms):X8}");
while (ms.Length < ms.Capacity)
{
ms.WriteByte(0);
}
vanillaFile.Position = 0;
ms.Position = 0;
BinaryReader original = new BinaryReader(vanillaFile);
BinaryReader test = new BinaryReader(ms);
sb.AppendLine($"{file.VRom} - original: {original.BaseStream.Length:X8} test: {test.BaseStream.Length:X8}");
for (int i = 0; i < file.Rom.Size; i += 4)
{
int left = original.ReadBigInt32();
int right = test.ReadBigInt32();
if (left != right)
{
sb.AppendLine($"{file.VRom} - {i:X8} does not match, comparison stopped");
}
}
}
catch (Exception e)
{
sb.AppendLine($"{file.VRom} - Exception {e.Message} {e.InnerException}");
}
}
face.OutputText(sb.ToString());
}
private static void WriteFile(BinaryReader file, FileRecord record, bool compress, string folder)
{
byte[] data;
using FileStream dest = new($"{record.VRom.Start}/{folder:X8}", FileMode.CreateNew);
data = new byte[record.VRom.Size];
file.Read(data, 0, record.VRom.Size);
if (compress)
{
Yaz.Encode(data, record.VRom.Size, dest);
}
else
{
dest.Write(data, 0, data.Length);
}
}
public static byte[] BuildROM()
{
// yaz0 encode all of the files for the rom
Parallel.ForEach(RomData.MMFileList, file =>
{
if (file.IsCompressed && file.WasEdited)
{
// lower priority so that the rando can't lock a badly scheduled CPU by using 100%
var previous_thread_priority = Thread.CurrentThread.Priority;
Thread.CurrentThread.Priority = ThreadPriority.Lowest;
byte[] result;
var newSize = Yaz.Encode(file.Data, file.Data.Length, out result);
if (newSize >= 0)
{
file.Data = new byte[newSize];
ReadWriteUtils.Arr_Insert(result, 0, newSize, file.Data, 0);
}
// this thread is borrowed, we don't want it to always be the lowest priority, return to previous state
Thread.CurrentThread.Priority = previous_thread_priority;
}
});
byte[] ROM = new byte[0x2000000];
int ROMAddr = 0;
// write all files to rom
for (int i = 0; i < RomData.MMFileList.Count; i++)
{
if (RomData.MMFileList[i].Cmp_Addr == -1)
{
continue;
}
RomData.MMFileList[i].Cmp_Addr = ROMAddr;
int file_len = RomData.MMFileList[i].Data.Length;
if (RomData.MMFileList[i].IsCompressed)
{
RomData.MMFileList[i].Cmp_End = ROMAddr + file_len;
}
ReadWriteUtils.Arr_Insert(RomData.MMFileList[i].Data, 0, file_len, ROM, ROMAddr);
ROMAddr += file_len;
}
UpdateFileTable(ROM);
SignROM(ROM);
FixCRC(ROM);
return(ROM);
}
private static Stream CompressFile(ORom rom, FileRecord record)
{
byte[] data;
using (BinaryReader br = new BinaryReader(rom.Files.GetFile(record.VRom.Start)))
{
data = br.ReadBytes(record.VRom.Size);
}
//compress file
MemoryStream ms = new MemoryStream();
Yaz.Encode(data, data.Length, ms);
ms.Position = 0;
return(ms);
}
private static void CompressRom(Rom rom, Dictionary <long, FileRecord> refFileList, Stream sw)
{
int filesProcessed = 0;
int filesTotal;
List <FileRecord> newDmaTable = new List <FileRecord>();
Console.WriteLine();
foreach (FileRecord record in rom.Files)
{
Stream outstream;
bool IsCompressed = false;
filesTotal = rom.Files.Count();
//get file
outstream = rom.Files.GetFile(record.VRom.Start);
//Compress if the file can't be found in the don't compress list,
//or if the file listed should be compressed (legacy)
if (!refFileList.ContainsKey(record.VRom.Start) ||
refFileList[record.VRom.Start].IsCompressed)
{
//compress file
IsCompressed = true;
MemoryStream ms = new MemoryStream();
using (BinaryReader br = new BinaryReader(outstream))
{
byte[] data = br.ReadBytes(record.VRom.Size);
Yaz.Encode(data, data.Length, ms);
ms.Position = 0;
}
outstream = ms;
}
//data contains the data to write to rom
AppendFile(sw, newDmaTable, outstream, record, IsCompressed);
//writes progress to console window
filesProcessed++;
Console.Write(processedFiles, filesProcessed, filesTotal);
}
sw.PadToLength(0x2000000);
WriteFileTable(sw, rom.Files.GetDmaDataStart(), newDmaTable);
CRC.Write(sw);
}
public static byte[] BuildROM(string FileName)
{
Parallel.ForEach(RomData.MMFileList, file =>
{
if (file.IsCompressed && file.WasEdited)
{
byte[] result;
var newSize = Yaz.Encode(file.Data, file.Data.Length, out result);
if (newSize >= 0)
{
file.Data = new byte[newSize];
ReadWriteUtils.Arr_Insert(result, 0, newSize, file.Data, 0);
}
}
});
byte[] ROM = new byte[0x2000000];
int ROMAddr = 0;
for (int i = 0; i < RomData.MMFileList.Count; i++)
{
if (RomData.MMFileList[i].Cmp_Addr == -1)
{
continue;
}
RomData.MMFileList[i].Cmp_Addr = ROMAddr;
int file_len = RomData.MMFileList[i].Data.Length;
if (RomData.MMFileList[i].IsCompressed)
{
RomData.MMFileList[i].Cmp_End = ROMAddr + file_len;
}
ReadWriteUtils.Arr_Insert(RomData.MMFileList[i].Data, 0, file_len, ROM, ROMAddr);
ROMAddr += file_len;
}
UpdateFileTable(ROM);
SignROM(ROM);
FixCRC(ROM);
using (BinaryWriter NewROM = new BinaryWriter(File.Open(FileName, FileMode.Create)))
{
NewROM.Write(ROM, 0, ROM.Length);
}
return(ROM);
}
public static int Compress(ReadOnlySpan <byte> read, Span <byte> w, CompressTask task, int cur = 0)
{
DmadataRecord dmadataRec = GetDmadataRec(read, task.Dmadata);
var dmadata = GetDmadataRecords(read, task.Dmadata);
for (int i = 0; i < dmadata.Count; i++)
{
if (i % 50 == 0)
{
Console.WriteLine($"File {i}");
}
var dmarec = dmadata[i];
FileAddress vrom = dmarec.VRom;
FileAddress rom = dmarec.Rom;
if (rom.Start < 0)
{
continue;
}
ReadOnlySpan <byte> file = read.Slice(rom.Start, vrom.Size);
//if file should be compressed
if (!task.Exclusions.Contains(i))
{
int size = Yaz.Encode(file.ToArray(), vrom.Size, out byte[] comp);
if (size > 0)
{
Span <byte> c = new(comp, 0, size);
c.CopyTo(w.Slice(cur, size));
dmarec.Rom = new FileAddress(cur, cur + size);
cur += size;
continue;
}
}
file.CopyTo(w.Slice(cur, vrom.Size));
dmarec.Rom = new FileAddress(cur, 0);
cur += vrom.Size;
}
WriteDmadataRecords(w.Slice(dmadataRec.Rom.Start, dmadataRec.VRom.Size), dmadata);
return(cur);
}
public static void TestYazSimple(IExperimentFace face, List <string> filePath)
{
StringBuilder sb = new StringBuilder();
ORom rom = new ORom(filePath[0], ORom.Build.N0);
RomFile rfile = rom.Files.GetFile(ORom.FileList.code);
var dmarec_code = rfile.Record;
BinaryReader compressed = new BinaryReader((MemoryStream)rom.Files.GetPhysicalFile(dmarec_code.VRom));
BinaryReader decompressed = new BinaryReader(rfile);
byte[] decompressedbuffer = decompressed.ReadBytes(rfile.Record.VRom.Size);
//var buffer = new MemoryStream(0x20_0000);
var buffer = new byte[0];
Stopwatch stopwatch = Stopwatch.StartNew();
//int compressedSize = Yaz.Encode(decompressedbuffer, rfile.Record.VirtualAddress.Size, buffer);
int compressedSize = Yaz.Encode(decompressedbuffer, rfile.Record.VRom.Size, out buffer);
Yaz.GetEncodingData(buffer, compressedSize, out int metaSize);
stopwatch.Stop();
sb.AppendLine($"Compression time: {stopwatch.Elapsed.Seconds}.{stopwatch.Elapsed.Milliseconds:D3}");
sb.AppendLine($"Compressed Size: {compressedSize:X6}, Meta Size {metaSize:X6}");
decompressed.Seek(0);
//buffer.Position = 0;
//compare compressed output
if (compressedSize != dmarec_code.Rom.Size)
{
sb.AppendLine($"Compression size mismatch: Original {dmarec_code.Rom.Size:X6} || New {compressedSize:X6}");
}
int mismatchMax = 8;
for (int i = 0; i < dmarec_code.Rom.Size; i++)
{
byte a = compressed.ReadByte();
//byte b = (byte)buffer.ReadByte();
byte b = buffer[i];
if (a == b)
{
continue;
}
mismatchMax--;
sb.AppendLine($"COMPRESS MISMATCH {i:X6}: {a:X2} {b:X2}");
if (mismatchMax <= 0)
{
break;
}
}
compressed.Seek(0);
//buffer.Position = 0;
byte[] dbuffer = Yaz.Decode(new MemoryStream(buffer), compressedSize);
decompressed.BaseStream.Position = 0;
for (int i = 0; i < dbuffer.Length; i++)
{
if (dbuffer[i] != decompressed.ReadByte())
{
sb.AppendLine($"File Size: {dbuffer.Length:X}, Compressed: {compressedSize:X}, Failed Match: {i:X}");
break;
}
}
sb.AppendLine("Test Complete");
face.OutputText(sb.ToString());
}
public byte[] EncodeNew()
{
var length = Yaz.Encode(CodeBytes, out var encoded);
return(encoded);
}