public void SaveMainDol(string outPath)
{
MemorySection hdr = this.NStream.DiscHeader;
this.NStream.Copy(ByteStream.Zeros, hdr.ReadUInt32B(0x420) - 0x440);
//dol info https://github.com/jchv/gcmtools/blob/master/gcm/dol.hpp
byte[] dolLen = new byte[4];
this.NStream.Read(dolLen, 0, 4);
MemorySection mx = new MemorySection(dolLen);
byte[] dolHdr = new byte[mx.ReadUInt32B(0)];
dolLen.CopyTo(dolHdr, 0);
this.NStream.Read(dolHdr, 4, dolHdr.Length - 4);
MemorySection maindol = new MemorySection(dolHdr);
uint maindolSize = 0;
for (int i = 0; i < 18; i++)
{
if (maindol.ReadUInt32B(0x0 + (i * 4)) != 0) //7 text offsets, 11 data offsets
{
maindolSize = Math.Max(maindolSize, maindol.ReadUInt32B(0x0 + (i * 4)) + maindol.ReadUInt32B(0x90 + (i * 4)));
}
}
byte[] dol = new byte[maindol.Size + maindolSize];
dolHdr.CopyTo(dol, 0);
this.NStream.Read(dol, dolHdr.Length, dol.Length - dolHdr.Length);
string dolName = Path.Combine(outPath, string.Format("main[{0}][{1}][{2}].dol", this.NStream.Id8, hdr.ReadUInt32B(0x420).ToString("X8"), Crc.Compute(dol).ToString("X8")));
if (!File.Exists(dolName))
{
File.WriteAllBytes(dolName, dol);
}
}
public void Populate()
{
if (!string.IsNullOrEmpty(this.Filename) && File.Exists(this.Filename))
{
MemorySection ms = new MemorySection(File.ReadAllBytes(this.Filename));
this.MainDolOffset = ms.ReadUInt32B(0x00);
this.FstOffset = ms.ReadUInt32B(0x04);
this.MaxFst = ms.ReadUInt32B(0x08);
this.Region = (Region)ms.ReadUInt32B(0x0C);
this.Title = ms.Read(0x10, 0x50 - 0x10);
this.FstData = ms.Read(0x50, (int)ms.Size - 0x50);
}
}
public HeaderBruteForcer(uint[] updateCrcs, SortedList <uint, bool> checkCrcs, Tuple <byte[], int[]>[] regionData, params byte[][] headers)
{
_updateCrcs = updateCrcs;
_checkCrcs = checkCrcs;
//DateTime dt = DateTime.Now;
_hdrs = headers.Select(a => new headerCrc()
{
Crc = Crc.Compute(a), CrcPreRegion = Crc.Compute(a, 0, 0x4e000), CrcRegion = Crc.Compute(a, 0x4e000, 0x20), CrcPostRegion = Crc.Compute(a, 0x4e020, 0x1fe0), Data = a
}).ToList();
MemorySection ms = new MemorySection(headers[0]);
_origRegion = (int)ms.ReadUInt32B(0x4e000);
_origRegionData = ms.Read(0x4e010, 0x10);
_origHeaderCrcs = _hdrs.Select(a => a.Crc).ToArray(); //original header Crcs
for (int h = _hdrs.Count - 1; h >= 1; h--)
{
if (_hdrs[h].Crc == _hdrs[h - 1].Crc)
{
_hdrs.RemoveAt(h); //remove duplicate headers
}
}
_regionData = new List <regionCrc>();
for (int h = 0; h < _hdrs.Count; h++)
{
for (int i = 0; i < regionData.Length; i++)
{
for (int r = 0; r < regionData[i].Item2.Length; r++)
{
_regionData.Add(new regionCrc()
{
Crc = 0, Header = _hdrs[h], RegionData = regionData[i].Item1, Region = (int)regionData[i].Item2[r]
});
}
}
}
//parallel precalculate the crcs for the matching
Parallel.ForEach(_regionData, rgn =>
{
MemorySection rg = new MemorySection(new byte[0x20]);
rg.WriteUInt32B(0, (uint)rgn.Region);
rg.Write(0x10, rgn.RegionData);
rgn.Crc = ~Crc.Combine(Crc.Combine(~rgn.Header.CrcPreRegion, ~Crc.Compute(rg.Data), 0x20), ~rgn.Header.CrcPostRegion, 0x1fe0);
});
//Trace.WriteLine(string.Format("Crc Headers: {0} - Took: {1}", _regionData.Count.ToString(), (DateTime.Now - dt).TotalMilliseconds.ToString()));
}
private static uint recurseFst(MemorySection ms, FstFolder folder, long names, uint i, string id, bool isGc)
{
uint j;
uint hdr = ms.ReadUInt32B((int)(12 * i));
long name = names + hdr & 0x00ffffffL;
int type = (int)(hdr >> 24);
string nm = ms.ReadStringToNull((int)name, Encoding.GetEncoding("shift-jis"));
uint size = ms.ReadUInt32B((int)(12 * i + 8));
if (type == 1)
{
FstFolder f = i == 0 ? folder : new FstFolder(folder)
{
Name = nm
};
if (i != 0)
{
folder.Folders.Add(f);
}
for (j = i + 1; j < size;)
{
j = recurseFst(ms, f, names, j, id, isGc);
}
return(size);
}
else
{
int pos = (int)(12 * i + 4);
long doff = ms.ReadUInt32B(pos) * (isGc ? 1L : 4L); //offset in data
size = ms.ReadUInt32B((int)(12 * i + 8));
long off = NStream.DataToOffset(doff, !isGc); //offset in raw partition
folder.Files.Add(new FstFile(folder)
{
DataOffset = doff, Offset = off, Length = size, Name = nm, PartitionId = id, OffsetInFstFile = pos
});
return(i + 1);
}
}
internal void Initialise(WiiPartitionGroupSection firstSection)
{
_firstSection = firstSection;
IsEncrypted = firstSection.IsEncrypted;
MemorySection ms = new MemorySection(firstSection.Decrypted);
this.Id = ms.ReadString(0x400, 4);
this.DiscNo = (int)ms.Read8(0x406);
if (this.Stream != null)
{
this.Stream.ChangeJunk(this.DiscOffset + _dataOffset, this.Id, this.DiscNo, PartitionDataSize);
}
if (this.Id != "\0\0\0\0")
{
_dolOffset = ms.ReadUInt32B(0x820) * 4L; //+400 to skip hashes
FstOffset = ms.ReadUInt32B(0x824) * 4L;
FstSize = ms.ReadUInt32B(0x828) * 4L;
}
}
internal static IEnumerable <WiiPartitionInfo> CreatePartitionInfos(MemorySection section, int offset)
{
for (int tableIdx = 0; tableIdx < 4; tableIdx++) //up to 4 partitions on the disk
{
uint c = section.ReadUInt32B(offset + (tableIdx * 8)); //count of partitions for tableIdx
//_log.Message(string.Format("Table {0} - Partitions {1}", tableIdx.ToString(), c.ToString("X8")), 2);
if (c == 0)
{
continue;
}
int tableOffset = (int)section.ReadUInt32B(offset + (tableIdx * 8) + 4) * 4; //first partition entry for tableIdx
int adjustReadOffset = offset + (tableOffset - _PartitionTableOffset);
for (int i = 0; i < c; i++)
{
long partitionOffset = section.ReadUInt32B(adjustReadOffset + (i * 8)) * 4L;
PartitionType partitionType = (PartitionType)section.ReadUInt32B(adjustReadOffset + (i * 8) + 4);
//_log.Message(string.Format(" PartitionOffset Offset {0} - Type {1}", partitionOffset.ToString("X8"), partitionType.ToString()), 2);
yield return(new WiiPartitionInfo(partitionType, partitionOffset, tableIdx, tableOffset + (i * 8)));
//_log.Message(string.Format(" ID {0}", partitions.Last().ReadStream.Id), 2);
}
}
}
public ExtractResult ExtractBasicInfo()
{
Region region;
if (this.IsGameCube)
{
MemorySection bi2bin = MemorySection.Read(this.NStream, 0x2000);
region = (Region)bi2bin.ReadUInt32B(0x18);
}
else
{
region = (Region)this.Header.ReadUInt32B(0x4e000);
}
return(createExtractResult(region, null));
}
internal static FileSystem Parse(MemorySection ms, FstFile fst, string id, bool isGc)
{
FstFolder fld = new FstFolder(null);
long nFiles = ms.ReadUInt32B(0x8);
if (12 * nFiles > ms.Size)
{
return(null);
}
if (fst != null)
{
fld.Files.Add(fst);
}
recurseFst(ms, fld, 12 * nFiles, 0, id, isGc);
return(new FileSystem(fld));
}
private long partitionWrite(Stream inStream, NCrc crc, Stream target, WiiPartitionSection pHdr, Context ctx, Coordinator pc, NkitInfo imageInfo, ScrubManager scrub, WiiHashStore hashes, long fstFileAlignment)
{
#if DEHASH
return(pHdr.NewPartitionDataLength = inStream.Copy(target, pHdr.PartitionDataLength, null));
#elif DECRYPT
inStream.Copy(target, pHdr.PartitionLength, null);
return(pHdr.NewPartitionDataLength = pHdr.PartitionDataLength);
#endif
MemorySection hdr = MemorySection.Read(inStream, 0x440);
//ProgressResult result = ctx.Result;
long mlt = 4L; //GC 1L
long logicalSize = pHdr.PartitionDataLength; //GC: ctx.ImageLength;
long physicalSize = pHdr.PartitionDataLength; //GC: result.ImageInfo.IsoSize;
string junkId = hdr.ReadString(0, 4);
//bool write = true;
List <string> addedFiles = new List <string>();
long srcPos;
long dstPos = 0;
JunkStream js = new JunkStream(junkId, hdr.Read8(6), logicalSize);
try
{
if (junkId == "\0\0\0\0")
{
srcPos = hdr.Size;
imageInfo.BytesData = srcPos;
_log?.LogDetail("Null Partition ID, preserving partition as raw");
ConvertFile cf = new ConvertFile(logicalSize - hdr.Size, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.Size, Offset = hdr.Size, Length = 0
},
};
long nullsPos = 0;
target.Write(hdr.Data, 0, (int)hdr.Size); //0x400
target.Write(pHdr.Header.Data, 0x2bc, 4); //copy the original partition length
dstPos += 0x440 + 4 + NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
hdr.WriteString(0x200, 8, "NKIT v01");
hdr.WriteUInt32B(0x210, (uint)(pHdr.PartitionLength / mlt));
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
crc.Snapshot(junkId + " PrtHdr");
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot(junkId + " Fst");
target.Write(hashes.FlagsToByteArray(), 0, (int)hashes.FlagsLength);
crc.Snapshot(junkId + " HashFlags");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos + hashes.FlagsLength;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, physicalSize, hdr, fst, false, fstFileAlignment, out error);
imageInfo.BytesData = srcPos;
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(pHdr.PartitionDataLength - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
List <ConvertFile> missing;
NkitFormat.NkitWriteFileSystem(ctx, imageInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out missing, scrub, pHdr.PartitionDataLength, _log);
dstPos += hashes.HashesToStream(target);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
crc.Crcs[crc.Crcs.Length - 1].PatchData = hashes.FlagsToByteArray();
crc.Crcs[crc.Crcs.Length - 1].PatchCrc = Crc.Compute(crc.Crcs[crc.Crcs.Length - 2].PatchData);
crc.Crcs[crc.Crcs.Length - 2].PatchData = fst.Data;
crc.Crcs[crc.Crcs.Length - 2].PatchCrc = Crc.Compute(fst.Data);
}
}
}
catch (Exception ex)
{
throw new HandledException(ex, "NkitWriterGc.Write - Convert");
}
return(dstPos);
}
public void Write(Context ctx, Stream input, Stream output, Coordinator pc)
{
try
{
long imageSize = pc.OutputSize;
string junkId;
pc.WriterCheckPoint1WriteReady(out junkId); //wait until read has written the header and set the length
MemorySection ms = new MemorySection(new byte[4 + 4 + 8 + 8 + 4 + 4]);
int blockSize = 0x4000;
int blocks = (int)(imageSize / blockSize) + (imageSize % blockSize == 0 ? 0 : 1);
ms.WriteUInt32L(0x00, 0xB10BC001);
ms.WriteUInt32L(0x04, 0); //insert NKIT later
ms.WriteUInt64L(0x08, 0); //size place holder
ms.WriteUInt64L(0x10, (ulong)imageSize);
ms.WriteUInt32L(0x18, (uint)blockSize);
ms.WriteUInt32L(0x1C, (uint)blocks);
MemorySection pnt = new MemorySection(new byte[blocks * 8]);
MemorySection hsh = new MemorySection(new byte[blocks * 4]);
long offset = 0;
NCrc crc = new NCrc();
CryptoStream target = new CryptoStream(output, crc, CryptoStreamMode.Write);
target.Write(ms.Data, 0, (int)ms.Size);
crc.Snapshot("Header");
target.Write(pnt.Data, 0, (int)pnt.Size);
crc.Snapshot("Pointers");
target.Write(hsh.Data, 0, (int)hsh.Size);
crc.Snapshot("Hashes");
long dataOffset = (pnt.Size + hsh.Size + ms.Size); //fso.position
int rdBlk = 0;
int wrBlk = 0;
object rdBlkLock = new object();
object wrBlkLock = new object();
Task[] tasks = new Task[3];
ManualResetEvent mr = new ManualResetEvent(false);
for (int i = 0; i < tasks.Length; i++) //4 threads
{
tasks[i] = (new TaskFactory()).StartNew((Object prm) =>
{
ManualResetEvent m = (ManualResetEvent)prm;
int blkIdx;
byte[] rawBlk = new byte[blockSize];
byte[] buff = new byte[blockSize + 0x100];
using (MemoryStream blk = new MemoryStream(buff))
{
while (true)
{
lock (rdBlkLock)
{
blkIdx = rdBlk++;
if (blkIdx >= blocks)
{
return; //no more processing
}
int read = input.Read(rawBlk, 0, (int)Math.Min(imageSize - (blkIdx * blockSize), blockSize));
if (read != blockSize)
{
Array.Clear(rawBlk, read, blockSize - read);
}
}
blk.Position = 0;
ZlibStream zl = new ZlibStream(blk, SharpCompress.Compressors.CompressionMode.Compress, SharpCompress.Compressors.Deflate.CompressionLevel.BestCompression, Encoding.Default);
zl.FlushMode = FlushType.Finish;
zl.Write(rawBlk, 0, blockSize);
zl.Flush();
while (blkIdx != wrBlk)
{
m.WaitOne();
}
mr.Reset();
if (blk.Position < blockSize)
{
target.Write(buff, 0, (int)blk.Position);
pnt.WriteUInt64L(blkIdx * 8, (ulong)(offset));
hsh.WriteUInt32L(blkIdx * 4, adler32(buff, (int)blk.Position));
offset += (int)blk.Position;
}
else
{
target.Write(rawBlk, 0, blockSize);
pnt.WriteUInt64L(blkIdx * 8, (ulong)(offset) | 0x8000000000000000);
hsh.WriteUInt32L(blkIdx * 4, adler32(rawBlk, blockSize));
offset += (int)blockSize;
}
target.Flush();
wrBlk++;
mr.Set();
}
}
}, mr);
}
Task.WaitAll(tasks);
crc.Snapshot("Files");
NCrc readerCrcs;
uint validationCrc;
pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, null, dataOffset + offset); //wait until reader has completed and get crc patches.
ms.WriteUInt64L(0x08, (ulong)offset);
crc.Crcs[0].PatchCrc = Crc.Compute(ms.Data);
crc.Crcs[0].PatchData = ms.Data;
crc.Crcs[1].PatchCrc = Crc.Compute(pnt.Data);
crc.Crcs[1].PatchData = pnt.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(hsh.Data);
crc.Crcs[2].PatchData = hsh.Data;
ms.WriteUInt32B(0x04, CrcForce.Calculate(crc.FullCrc(true), output.Length, readerCrcs.FullCrc(true), 0x04, ms.ReadUInt32B(0x04))); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(ms.Data);
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "GCZ Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "GczWriter.Write - Compress");
}
}
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
{
try
{
long mlt = 1L; //for Wii: 4L
long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength
pc.WriterCheckPoint1WriteReady(out string junkId); //wait until read has written the header and set the length
List <string> addedFiles = new List <string>();
NCrc crc = new NCrc();
long srcPos;
long dstPos = 0;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
string id8 = string.Concat(hdr.ReadString(0, 6), hdr.Data[6].ToString("X2"), hdr.Data[7].ToString("X2"));
if (junkId == null)
{
junkId = ctx.Settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == id8)?.JunkId;
if (junkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", junkId, id8.Substring(0, 4)));
}
}
if (junkId == null)
{
junkId = hdr.ReadString(0, 4);
}
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
long fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == id8)?.Item2 ?? -1;
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
crc.Snapshot("hdr.bin");
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot("fst.bin");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
JunkStream js = new JunkStream(junkId, hdr.Read8(6), NStream.FullSizeGameCube);
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, imageSize, hdr, fst, true, fstFileAlignment, out string error); //Size isn't important for writing //result.ImageInfo.IsoSize
NkitInfo nkitInfo = new NkitInfo
{
BytesData = srcPos,
BytesGaps = 0,
BytesJunkFiles = 0,
BytesPreservationData = 0,
BytesPreservationDiscPadding = 0
};
ScrubManager scrub = new ScrubManager();
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
NkitFormat.NkitWriteFileSystem(ctx, nkitInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out List <ConvertFile> missing, scrub, imageSize, _log);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
}
if (dstPos % 0x800 != 0)
{
long l = 0x800 - (dstPos % 0x800);
ByteStream.Zeros.Copy(target, l);
dstPos += l;
nkitInfo.BytesPreservationDiscPadding += l;
}
crc.Snapshot("files");
NkitFormat.LogNkitInfo(nkitInfo, _log, hdr.ReadString(0, 4), true);
pc.WriterCheckPoint2Complete(out NCrc readerCrcs, out uint validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.
hdr.WriteUInt32B(0x420, (uint)mainDolAddr);
hdr.WriteString(0x200, 8, "NKIT v01"); //header and version
hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
hdr.WriteUInt32B(0x210, (uint)imageSize); //result.ImageInfo.IsoSize);
hdr.WriteString(0x214, 4, hdr.ReadString(0, 4) != junkId ? junkId : "\0\0\0\0");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, "NKit Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "NkitWriterGc.Write - Convert");
}
}
internal static List <ConvertFile> GetConvertFstFiles(Stream inStream, long size, MemorySection hdr, MemorySection fst, bool isGc, long fstFileAlignment, out string error)
{
string[] align = new string[] { ".tgc" }; //, ".dol", ".thp", ".adp", ".poo", ".pcm", ".pcm16", ".son", ".act", ".bin", ".gpl", ".tpl", ".bmp", ".sam", ".sdi", ".tcs", ".txt", ".skn", ".stp", ".anm" };
error = null;
List <ConvertFile> conFiles = new List <ConvertFile>();
try
{
List <FstFile> srcFiles = FileSystem.Parse(fst, null, hdr.ReadString(0, 4), isGc)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
//get list of files and gaps
long end;
long gap;
long fstLen = hdr.ReadUInt32B(0x424) * (isGc ? 1L : 4L);
FstFile ff = new FstFile(null)
{
Name = "fst.bin", DataOffset = fstLen, Offset = fstLen, Length = (int)fst.Size, IsNonFstFile = true
};
for (int i = 0; i < srcFiles.Count; i++)
{
ff = i == 0 ? ff : srcFiles[i - 1];
end = ff.DataOffset + ff.Length;
end += end % 4 == 0 ? 0 : 4 - (end % 4);
gap = srcFiles[i].DataOffset - end;
if (gap < 0)
{
error = string.Format("The gap between '{0}' and '{1}' is {2} - Converting as bad image", ff.Name, srcFiles[i].Name, gap.ToString());
return(null);
}
conFiles.Add(new ConvertFile(gap, isGc)
{
FstFile = ff
});
}
ff = srcFiles.Last();
end = ff.DataOffset + ff.Length;
end += end % 4 == 0 ? 0 : 4 - (end % 4);
gap = size - end;
if (gap >= -3 && gap < 0)
{
gap = 0; //some hacked gc images converted from tgc end on the file end (star fox e3)
}
if (gap < 0)
{
error = string.Format("The gap between '{0}' and the end of the image is {1} - Converting as bad image/partition", ff.Name, gap.ToString());
return(null);
}
conFiles.Add(new ConvertFile(gap, isGc)
{
FstFile = ff
});
//set alignment
foreach (ConvertFile cf in conFiles)
{
ff = cf.FstFile;
if (fstFileAlignment == 0)
{
cf.Alignment = 0; //preserve alignment
}
else if (fstFileAlignment == -1 && ff.DataOffset % 0x8000 == 0 && (ff.Length % 0x8000 == 0 || align.Contains(Path.GetExtension(ff.Name).ToLower())))
{
cf.Alignment = 0x8000; //default behaviour
}
else if (fstFileAlignment != 0 && ff.DataOffset % fstFileAlignment == 0) //src matches alignment
{
cf.Alignment = fstFileAlignment; //align to largest multiple
}
else
{
cf.Alignment = -1; //-1 = do not align this file
}
}
}
catch
{
error = "Fst parsing error - Converting as bad image";
return(null);
}
return(conFiles);
}
internal static void NkitWriteFileSystem(Context ctx, NkitInfo imageInfo, long mlt, Stream inStream, ref long srcPos, ref long dstPos, MemorySection hdr, MemorySection fst, ref long mainDolAddr, Stream target, long nullsPos, JunkStream js, List <ConvertFile> conFiles, out List <ConvertFile> missingFiles, ScrubManager scrub, long imageSize, ILog log)
{
FstFile ff;
//########### FILES
bool firstFile = true;
ConvertFile lastf = conFiles.Last();
ConvertFile prevf = null;
missingFiles = new List <ConvertFile>();
foreach (ConvertFile f in conFiles) //read the files and write them out as goodFiles (possible order difference
{
ff = f.FstFile;
if (!firstFile)
{
if (srcPos == mainDolAddr && mainDolAddr == (hdr.ReadUInt32B(0x420) * mlt))
{
mainDolAddr = dstPos; //main/default.dol is moving
}
//Debug.WriteLine(string.Format(@"{5} : {0} : {1} : {2} : {3}/{4}", ff.DataOffset.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), /*(nextFile.DataOffset - lastEnd).ToString("X8"),*/ ff.Length.ToString("X8"), ff.Path, ff.Name, ff.OffsetInFstFile.ToString("X8")));
//srcPos aligned at 32k and the length is 32k (audio file) ensure new dest is also aligned
if (f.Alignment == 0 || (f.Alignment != -1 && dstPos % f.Alignment != 0)) //XGIII audio in race, Zelda collection games for TGC
{
long pad;
if (f.Alignment == 0) //no shrinking
{
pad = Math.Max(0, ff.DataOffset - dstPos);
}
else //align
{
pad = (int)(dstPos % f.Alignment == 0 ? 0 : f.Alignment - (dstPos % f.Alignment));
}
imageInfo.FilesAligned += 1;
ByteStream.Zeros.Copy(target, pad);
imageInfo.BytesPreservationDiscPadding += pad;
dstPos += pad;
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(dstPos / mlt));
//replace copy with junk test - adjust fst length to %4
long l = srcPos;
imageInfo.FilesTotal += 1;
long written = NkitFormat.CopyFile(ref nullsPos, f, prevf?.FstFile, target, ref srcPos, dstPos, inStream, js, imageSize, out bool missing);
if (missing)
{
missingFiles.Add(f);
}
dstPos += written;
imageInfo.BytesData += srcPos - l; //read
if (f.Gap.JunkFile != 0) //encode the gap and write - pad to %4
{
imageInfo.BytesJunkFiles += f.Gap.JunkFile + (f.Gap.JunkFile % 4 == 0 ? 0 : (4 - (f.Gap.JunkFile % 4)));
fst.WriteUInt32B(ff.OffsetInFstFile + 4, 0); //modify size to be remainder of 4
}
}
if (f.GapLength != 0 || f.Gap.JunkFile != 0)
{
long l = NkitFormat.ProcessGap(ref nullsPos, f, ref srcPos, inStream, js, firstFile || f == lastf, scrub, target, log);
imageInfo.BytesGaps += f.GapLength;
imageInfo.BytesPreservationData += l;
dstPos += l;
}
firstFile = false;
prevf = f;
}
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
long imageSize = hdr.ReadUInt32B(0x210) * 4L;
string junkId = hdr.ReadString(0x214, 4);
uint updatePartitionCrc = hdr.ReadUInt32B(0x218);
MemorySection updateRemovedFiller = null;
long discOffset = 0;
List <NkitPartitionPatchInfo> patchInfos = new List <NkitPartitionPatchInfo>();
discOffset += hdr.Size;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
NCrc crc = new NCrc();
long dstPos = 0;
long srcPos = hdr.Size;
ScrubManager scrubFiller = new ScrubManager(null);
bool isRecoverable = false;
using (NDisc disc = new NDisc(_log, inStream))
{
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
hdr.Write8(0x60, 0);
hdr.Write8(0x61, 0);
CryptoStream crcStream = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //wrap to calculate crc
crcStream.Write(hdr.Data, 0, hdr.Data.Length); //write the header
pc.ReaderCheckPoint1PreWrite(null, nkitCrc); //size that will be output from this read
dstPos += hdr.Size;
crc.Snapshot("Disc Header");
foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted
{
if (updatePartitionCrc != 0 && srcPos == hdr.Size) //write update partition out
{
updateRemovedFiller = MemorySection.Read(inStream, hdr.Partitions.First().DiscOffset - srcPos);
srcPos += updateRemovedFiller.Size;
WiiPartitionInfo firstPart = WiiDiscHeaderSection.CreatePartitionInfos(updateRemovedFiller, 0)?.FirstOrDefault(a => a.Type != PartitionType.Update);
string updateFileName = RecoveryData.GetUpdatePartition(ctx.Settings, updatePartitionCrc);
if (updateFileName != null)
{
using (FileStream pf = File.OpenRead(updateFileName))
{
pf.Copy(crcStream, pf.Length);
dstPos += pf.Length;
}
}
else
{
string msg = string.Format("!! Update partition *_{0} missing - Adding filler. It may be Recoverable", updatePartitionCrc.ToString("X8"));
isRecoverable = true;
_log?.LogDetail(msg);
//throw pc.SetReaderException(new HandledException("Failed to convert: " + msg));
}
ByteStream.Zeros.Copy(crcStream, firstPart.DiscOffset - dstPos); //fill full gap
dstPos += firstPart.DiscOffset - dstPos;
}
NkitPartitionPatchInfo patchInfo = new NkitPartitionPatchInfo()
{
HashGroups = new Dictionary <long, MemorySection>()
};
patchInfos.Add(patchInfo);
if (part.DiscOffset > srcPos)
{
dstPos += writeFiller(ref srcPos, dstPos, dstPos + 0x1cL, inStream, crcStream, new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize), scrubFiller);
inStream.Copy(ByteStream.Zeros, part.DiscOffset - srcPos); //padded to 0x8000
srcPos += part.DiscOffset - srcPos;
}
part.DiscOffset = dstPos; //restore the original position
patchInfo.DiscOffset = dstPos;
patchInfo.PartitionHeader = MemorySection.Read(inStream, 0x20000);
srcPos += patchInfo.PartitionHeader.Size;
long size = patchInfo.PartitionHeader.ReadUInt32B(0x2bc) * 4L;
LongRef origSize = new LongRef()
{
Value = 0
};
WiiPartitionGroupSection wp = null;
WiiPartitionHeaderSection wh = new WiiPartitionHeaderSection(hdr, null, part.DiscOffset, patchInfo.PartitionHeader.Data, patchInfo.PartitionHeader.Data.Length);
MemorySection ph = new MemorySection(new byte[0x8000 * 64]);
long remaining = long.MaxValue;//set after first block read
int groupIndex = 0;
WiiHashStore hashes = new WiiHashStore();
patchInfo.ScrubManager = wh.ScrubManager;
bool patchBlock = false;
StreamCircularBuffer prtStream = null;
try
{
using (prtStream = new StreamCircularBuffer(0, null, null, output => srcPos += partitionStreamWrite(origSize, inStream, output, size, ctx.Dats, patchInfo, hashes, pc)))
{
int gs = 0;
int ge = 0;
int i = 0;
MemoryStream patchBlocks = null;
while (remaining > 0)
{
int blocks = (int)Math.Min(64L, remaining / 0x7c00);
for (int b = 0; b < blocks; b++)
{
prtStream.Read(ph.Data, (b * 0x8000) + 0x400, 0x7c00); //load aligned with no hashes
if (remaining == long.MaxValue) //first loop
{
remaining = origSize.Value;
if (ph.ReadString(0x400 + 0, 4) == "\0\0\0\0")
{
gs = -1;
ge = -1;
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, (uint)(NStream.DataToHashedLen(origSize.Value) / 4)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
}
else
{
gs = (int)((ph.ReadUInt32B(0x400 + 0x424) * 4L) / (0x7c00L * 64));
ge = (int)(((ph.ReadUInt32B(0x400 + 0x424) * 4L) + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) / (0x7c00L * 64));
if ((int)((part.DiscOffset + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) % (0x7c00L * 64)) == 0)
{
ge--; //don't load the next group if the data will end on the last byte
}
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, ph.ReadUInt32B(0x400 + 0x210)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
ph.WriteUInt32B(0x400 + 0x200, 0);
ph.WriteUInt32B(0x400 + 0x204, 0);
ph.WriteUInt32B(0x400 + 0x208, 0);
ph.WriteUInt32B(0x400 + 0x20C, 0);
ph.WriteUInt32B(0x400 + 0x210, 0);
ph.WriteUInt32B(0x400 + 0x214, 0);
ph.WriteUInt32B(0x400 + 0x218, 0);
}
wp = new WiiPartitionGroupSection(hdr, wh, ph.Data, part.DiscOffset, blocks * 0x8000, false);
wh.Initialise(wp, origSize.Value);
}
}
if (blocks < 64)
{
Array.Clear(ph.Data, blocks * 0x8000, ph.Data.Length - (blocks * 0x8000)); //clear remaining blocks
}
wp.Populate(groupIndex, ph.Data, dstPos, blocks * 0x8000);
int scrubbed = 0;
for (int bi = 0; bi < blocks; bi++)
{
wp.MarkBlockDirty(bi);
byte byt;
if (patchInfo.ScrubManager.IsBlockScrubbedScanMode(wp.Offset + (bi * 0x8000), out byt))
{
wp.SetScrubbed(bi, byt);
scrubbed++;
}
}
bool isFstBlocks = i >= gs && i <= ge;
bool reqHashes = hashes.IsPreserved(wp.Offset); //test with 0 partition based offset
repairBlocks(wp, scrubbed, blocks, false, isFstBlocks); //only test if the hashes aren't preserved (only preserved for scrubbed/customs)
if (reqHashes) //store with disc based offset
{
patchInfo.HashGroups.Add(wp.Offset + part.DiscOffset + patchInfo.PartitionHeader.Size, new MemorySection((byte[])wp.Decrypted.Clone())); //fetch the stored hashed that couldn't be recreated
}
groupIndex++;
bool inFstArea = i >= gs && i <= ge;
if (!patchBlock && (gs == i || reqHashes))
{
patchBlocks = new MemoryStream();
crc.Snapshot(lastPartitionId + " Data");
patchBlock = true;
}
else if (patchBlock && !inFstArea && !reqHashes)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
patchBlock = false;
}
#if DECRYPT
outStream.Write(wp.Decrypted, 0, blocks * 0x8000);
if (i >= gs && i <= ge)
{
fstBlocks.Write(wp.Decrypted, 0, blocks * 0x8000);
}
#else
crcStream.Write(wp.Encrypted, 0, blocks * 0x8000);
if (patchBlock)
{
patchBlocks.Write(wp.Encrypted, 0, blocks * 0x8000);
}
#endif
remaining -= (blocks * 0x7c00);
dstPos += (blocks * 0x8000);
i++;
}
if (patchBlock)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
}
}
if (origSize.Value != prtStream.WriterPosition)
{
throw pc.SetReaderException(new HandledException("Partition read did not write the full amount to the circular buffer before completing"));
}
}
catch (Exception ex)
{
if (prtStream?.WriterException != null)
{
throw pc.SetReaderException(prtStream.WriterException, "Failed reading filesystem");
}
throw pc.SetReaderException(ex, "Failed converting the filesystem");; //writer exception
}
srcPos += hashes.ReadPatchData(part.DiscOffset + patchInfo.PartitionHeader.Size, patchInfo.HashGroups, inStream);
//read hash patches
lastPartitionType = part.Type;
}
if (srcPos < inStream.Length)
{
JunkStream partJunk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
dstPos += writeFiller(ref srcPos, dstPos, lastPartitionType == PartitionType.Update ? imageSize : dstPos + 0x1cL, inStream, crcStream, partJunk, scrubFiller);
}
}
crc.Snapshot("End");
if (updatePartitionCrc != 0)
{
hdr.Write((int)WiiDiscHeaderSection.PartitionTableOffset, updateRemovedFiller.Data, 0, (int)WiiDiscHeaderSection.PartitionTableLength); //restore the exact partition table if update was removed
}
else
{
hdr.UpdateOffsets(); //just update the table with the new offsets
}
crc.Crcs[0].PatchData = hdr.Data;
foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
{
NkitPartitionPatchInfo patchInfo = patchInfos.FirstOrDefault(a => ci.Offset >= a.DiscOffset + a.PartitionHeader.Size && ci.Offset < a.DiscOffset + a.PartitionHeader.Size + a.Size);
if (patchInfo != null)
{
patchGroups(patchInfo, hdr, ci.Offset, ci.PatchData);
}
ci.PatchCrc = Crc.Compute(ci.PatchData);
}
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, isRecoverable, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - Read and convert");
}
}
private bool bruteForceValidHeader(Context ctx, FstFileItem goodFst, MemorySection hdr, MemorySection bi2bin, MemorySection appldr, MemorySection maindol, MemorySection fst, Coordinator pc)
{
int hdrScrub = 0;
int bi2Scrub = 0;
int maindolScrub = 0;
foreach (int scrub in new int[] { 0, 1, 2 })
{
if (scrub == 1)
{
hdrScrub = scrubData(hdr.Data, 0x300, 4);
bi2Scrub = scrubData(bi2bin.Data, 0x500 - 0x440, 4);
if (hdrScrub == 0 && bi2Scrub == 0)
{
continue; //no change
}
}
if (scrub == 2)
{
hdrScrub = scrubData(hdr.Data, 0x80, 0x3F0 - 0x80);
bi2Scrub = scrubData(bi2bin.Data, 0x30, bi2bin.Data.Length - (0x30 + 0x60)); //0x30 at start and 0x50 from end (conflict I, II + Splinter cell have 0x40 at end)
if (maindol.Size != 0)
{
uint maindolSize = 0;
for (int i = 0; i < 18; i++)
{
if (maindol.ReadUInt32B(0x0 + (i * 4)) != 0) //7 text offsets, 11 data offsets
{
maindolSize = Math.Max(maindolSize, maindol.ReadUInt32B(0x0 + (i * 4)) + maindol.ReadUInt32B(0x90 + (i * 4)));
}
}
maindolScrub = scrubData(maindol.Data, (int)maindolSize, (int)(maindol.Data.Length - maindolSize));
}
if (hdrScrub == 0 && bi2Scrub == 0 && maindolScrub == 0)
{
continue; //no change
}
}
NCrc crc = new NCrc();
MemoryStream ms = new MemoryStream();
using (CryptoStream target = new CryptoStream(ms, crc, CryptoStreamMode.Write))
{
target.Write(hdr.Data, 0, (int)hdr.Size);
long dstPos = hdr.Data.Length;
target.Write(bi2bin.Data, 0, (int)bi2bin.Size);
dstPos += bi2bin.Size;
target.Write(appldr.Data, 0, (int)appldr.Size);
dstPos += appldr.Size;
ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), goodFst.FstOffset) - dstPos);
dstPos += Math.Min(hdr.ReadUInt32B(0x420), goodFst.FstOffset) - dstPos;
target.Write(maindol.Data, 0, Math.Min((int)(goodFst.FstOffset - dstPos), (int)maindol.Size));
dstPos += Math.Min((int)(goodFst.FstOffset - dstPos), (int)maindol.Size);
long padding = goodFst.FstOffset - dstPos;
ByteStream.Zeros.Copy(target, padding);
dstPos += padding;
target.Write(fst.Data, 0, fst.Data.Length);
dstPos += fst.Size;
Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>();
crc.Snapshot("header");
}
if (crc.FullCrc() == goodFst.PostFstCrc)
{
if (hdrScrub != 0 || bi2Scrub != 0 || maindolScrub != 0)
{
//ctx.Result.GameCube.UpdatedHeader = true;
_log?.LogDetail("Header brute forced to match");
if (scrub == 1)
{
_log?.LogDetail(string.Format(" Bytes scrubbed at 0x300 and 0x500", hdrScrub.ToString(), bi2Scrub.ToString()));
}
else if (scrub == 2)
{
_log?.LogDetail(string.Format(" Bytes scrubbed in hdr.bin ({0}), bi2.bin ({1}), main.dol ({2})", hdrScrub.ToString(), bi2Scrub.ToString(), maindolScrub.ToString()));
}
}
return(true);
}
}
return(false);
}
private long partitionStreamWrite(LongRef outSize, Stream inStream, Stream target, long size, DatData settingsData, NkitPartitionPatchInfo patchInfo, WiiHashStore hashes, Coordinator pc)
{
DatData data = settingsData;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
long srcPos = hdr.Size;
long outPos = 0;
long imageSize = 0;
try
{
if (hdr.ReadString(0, 4) == "\0\0\0\0")
{
long nullsPos = 0;
long fileLength = -1;
LongRef gapLength = new LongRef()
{
Value = -1
};
target.Write(hdr.Data, 0, (int)hdr.Size);
MemorySection sz = MemorySection.Read(inStream, 4);
srcPos += 4;
outPos += hdr.Size;
imageSize = sz.ReadUInt32B(0) * 4L;
outSize.Value = NStream.HashedLenToData(imageSize);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), outSize.Value); //SET LENGTH FROM HEADER
outPos += writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
imageSize = NStream.HashedLenToData((hdr.ReadUInt32B(0x210) * 4L));
outSize.Value = imageSize;
string junkId = hdr.ReadString(0x214, 4);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), imageSize); //SET LENGTH FROM HEADER
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x424) * 4L) - hdr.Size);
srcPos += hdrToFst.Size;
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) * 4L);
long postFstPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
srcPos += fst.Size;
hashes.WriteFlagsData(imageSize, inStream);
srcPos += hashes.FlagsLength;
patchInfo.PartitionDataHeader = hdr;
patchInfo.Fst = fst;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
target.Write(fst.Data, 0, fst.Data.Length);
hdrToFst = null; //let this be collected if needed
outPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
long nullsPos = outPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, size, hdr, fst, false, -1, out error);
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
outPos += writeGap(cf, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
conFiles[0].GapLength -= hashes.FlagsLength; //fix for a few customs (no gap between the fst and the first file on the source image, but the hash mask makes it look like there is)
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), outPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (outPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset) //skip any padding (not written for wii currently)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)(outPos / 4L));
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(outPos / 4L));
outPos += copyFile(f, ref nullsPos, ref srcPos, outPos, inStream, target);
}
if (outPos < imageSize)
{
long gapLen = writeGap(f, ref nullsPos, ref srcPos, outPos, inStream, target, junk, i == 0 || i == conFiles.Count - 1, patchInfo.ScrubManager);
outPos += gapLen;
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)(ff.Length));
}
}
firstFile = false;
}
}
}
return(srcPos);
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - partitionRead");
}
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
DatData data = ctx.Dats;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
uint imageSize = hdr.ReadUInt32B(0x210);
string junkId = hdr.ReadString(0x214, 4);
if (junkId != "\0\0\0\0")
{
inStream.ChangeJunk(junkId);
}
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
NCrc crc = new NCrc();
NStream target = new NStream(new CryptoStream(outStream, crc, CryptoStreamMode.Write));
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - hdr.Size);
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) + (hdr.ReadUInt32B(0x428) % 4 == 0 ? 0 : 4 - (hdr.ReadUInt32B(0x428) % 4)));
long srcPos = hdr.ReadUInt32B(0x424) + fst.Size;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(junkId, hdr.ReadUInt32B(0x208)); //size that we will output from this read
crc.Snapshot("hdr.bin");
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
crc.Snapshot("fst.bin");
hdrToFst = null; //let this be collected if needed
long dstPos = hdr.ReadUInt32B(0x424) + fst.Size;
long nullsPos = dstPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, inStream.Length, hdr, fst, true, -1, out error); //result.ImageInfo.IsoSize
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(inStream.Length - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += writeGap(cf, ref nullsPos, ref srcPos, dstPos, inStream, target, true);
}
else
{
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), dstPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (dstPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)dstPos);
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)dstPos);
dstPos += copyFile(f, ref nullsPos, ref srcPos, dstPos, imageSize, inStream, target);
}
if (dstPos < imageSize)
{
dstPos += writeGap(f, ref nullsPos, ref srcPos, dstPos, inStream, target, i == 0 || i == conFiles.Count - 1);
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)ff.Length);
}
}
firstFile = false;
}
}
crc.Snapshot("files");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, false, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderGc.Read - Read and convert"); //don't let the writer lock
}
}
private long writeGap(ConvertFile file, ref long nullsPos, ref long srcPos, long dstPos, NStream inStream, Stream target, bool firstOrLastFile)
{
if (file.GapLength == 0)
{
if (file.FstFile.Length == 0)
{
nullsPos = dstPos + 0x1c;
}
return(0);
}
MemorySection ms = MemorySection.Read(inStream, 4);
srcPos += 4;
long size = ms.ReadUInt32B(0);
GapType gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
if (size == 0xFFFFFFFC) //for wii only. not a thing for GC
{
srcPos += 4;
inStream.Read(ms.Data, 0, 4);
size = 0xFFFFFFFCL + (long)ms.ReadUInt32B(0); //cater for files > 0xFFFFFFFF
}
long nulls;
long junkFileLen = 0;
//set nullsPos value if zerobyte file without junk
if (gt == GapType.JunkFile)
{
nullsPos = Math.Min(nullsPos - dstPos, 0);
nulls = (size & 0xFC) >> 2;
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
junkFileLen = ms.ReadUInt32B(0);
file.FstFile.Length = junkFileLen;
junkFileLen += junkFileLen % 4 == 0 ? 0 : 4 - (junkFileLen % 4);
ByteStream.Zeros.Copy(target, nulls);
inStream.JunkStream.Position = dstPos + nulls;
inStream.JunkStream.Copy(target, junkFileLen - nulls);
dstPos += junkFileLen;
if (file.GapLength <= 8)
{
return(junkFileLen);
}
else
{
//read gap
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
size = ms.ReadUInt32B(0);
gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
}
}
else if (file.FstFile.Length == 0) //last zero byte file was legit
{
nullsPos = dstPos + 0x1c;
}
long maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (size < maxNulls) //need to test this commented if
{
nulls = size;
}
else
{
nulls = size >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
if (gt == GapType.AllJunk)
{
ByteStream.Zeros.Copy(target, nulls);
inStream.JunkStream.Position = dstPos + nulls;
inStream.JunkStream.Copy(target, size - nulls);
dstPos += size;
}
else if (gt == GapType.AllScrubbed)
{
ByteStream.Zeros.Copy(target, size);
dstPos += size;
}
else
{
long prg = size;
byte btByte = 0x00;
GapBlockType bt = GapBlockType.Junk; //should never be used
while (prg > 0)
{
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
long bytes;
long blk = ms.ReadUInt32B(0);
GapBlockType btType = (GapBlockType)(blk >> 30);
bool btRepeat = btType == GapBlockType.Repeat;
if (!btRepeat)
{
bt = btType;
}
long cnt = 0x3FFFFFFF & blk;
if (bt == GapBlockType.NonJunk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
inStream.Copy(target, bytes);
srcPos += bytes;
}
else if (bt == GapBlockType.ByteFill)
{
if (!btRepeat)
{
btByte = (byte)(0xFF & cnt); //last 8 bits when not repeating are the byte
cnt >>= 8;
}
bytes = Math.Min(cnt * Gap.BlockSize, prg);
Stream bs;
switch (btByte)
{
case 0x00: bs = ByteStream.Zeros; break;
case 0x55: bs = ByteStream.Fives; break;
case 0xff: bs = ByteStream.FFs; break;
default: bs = new ByteStream(btByte); break;
}
bs.Copy(target, bytes);
}
else //if (bt == GapBlockType.Junk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (prg < maxNulls)
{
nulls = bytes;
}
else
{
nulls = bytes >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
ByteStream.Zeros.Copy(target, nulls);
inStream.JunkStream.Position = dstPos + nulls;
inStream.JunkStream.Copy(target, bytes - nulls);
}
prg -= bytes;
dstPos += bytes;
}
}
return(size + junkFileLen);
}
private long writeGap(ref long fileLength, LongRef gapLength, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target, JunkStream junk, bool firstOrLastFile, ScrubManager scrub)
{
if (gapLength.Value == 0)
{
if (fileLength == 0)
{
nullsPos = dstPos + 0x1c;
}
return(0);
}
long srcLen = gapLength.Value; //fix added for (padding between junk files) - Zumba Fitness (Europe) (En,Fr,De,Es,It)
MemorySection ms = MemorySection.Read(inStream, 4);
srcPos += 4;
long size = ms.ReadUInt32B(0);
GapType gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
if (size == 0xFFFFFFFC) //for wii only. not a thing for GC
{
srcPos += 4;
inStream.Read(ms.Data, 0, 4);
size = 0xFFFFFFFCL + (long)ms.ReadUInt32B(0); //cater for files > 0xFFFFFFFF
}
gapLength.Value = size;
scrub.AddGap(fileLength, dstPos, size); //keep track of trailing nulls when restoring scrubbed images
long nulls;
long junkFileLen = 0;
//set nullsPos value if zerobyte file without junk
if (gt == GapType.JunkFile)
{
nullsPos = Math.Min(nullsPos - dstPos, 0);
nulls = (size & 0xFC) >> 2;
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
junkFileLen = ms.ReadUInt32B(0);
fileLength = junkFileLen;
junkFileLen += junkFileLen % 4 == 0 ? 0 : 4 - (junkFileLen % 4);
ByteStream.Zeros.Copy(target, nulls);
junk.Position = dstPos + nulls;
junk.Copy(target, junkFileLen - nulls);
dstPos += junkFileLen;
if (srcLen <= 8)
{
return(junkFileLen);
}
else
{
//read gap
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
size = ms.ReadUInt32B(0);
gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
gapLength.Value = size;
}
}
else if (fileLength == 0) //last zero byte file was legit
{
nullsPos = dstPos + 0x1c;
}
long maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (size < maxNulls) //need to test this commented if
{
nulls = size;
}
else
{
nulls = size >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
nullsPos = dstPos + nulls; //belt and braces
if (gt == GapType.AllJunk)
{
ByteStream.Zeros.Copy(target, nulls);
junk.Position = dstPos + nulls;
junk.Copy(target, size - nulls);
dstPos += size;
}
else if (gt == GapType.AllScrubbed)
{
scrub.Scrub(target, dstPos, size, 0);
dstPos += size;
}
else
{
long prg = size;
byte btByte = 0x00;
GapBlockType bt = GapBlockType.Junk; //should never be used
while (prg > 0)
{
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
long bytes;
long blk = ms.ReadUInt32B(0);
GapBlockType btType = (GapBlockType)(blk >> 30);
bool btRepeat = btType == GapBlockType.Repeat;
if (!btRepeat)
{
bt = btType;
}
long cnt = 0x3FFFFFFF & blk;
if (bt == GapBlockType.NonJunk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
inStream.Copy(target, bytes);
srcPos += bytes;
}
else if (bt == GapBlockType.ByteFill)
{
if (!btRepeat)
{
btByte = (byte)(0xFF & cnt); //last 8 bits when not repeating are the byte
cnt >>= 8;
}
bytes = Math.Min(cnt * Gap.BlockSize, prg);
scrub.Scrub(target, dstPos, bytes, btByte);
}
else //if (bt == GapBlockType.Junk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (prg < maxNulls)
{
nulls = bytes;
}
else
{
nulls = bytes >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
ByteStream.Zeros.Copy(target, nulls);
junk.Position = dstPos + nulls;
junk.Copy(target, bytes - nulls);
}
prg -= bytes;
dstPos += bytes;
}
}
return(gapLength.Value + junkFileLen);
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
string resultMsg = "";
if (!Settings.ConfigFileFound)
{
_log?.Log("!! No config file found - This is required to restore and validate images");
}
NCrc crc = new NCrc();
Settings settings = ctx.Settings;
List <string> addedFiles = new List <string>();
DatData data = ctx.Dats;
RecoveryData rec = ctx.Recovery;
List <JunkRedumpPatch> patches = rec.JunkPatches;
MemorySection hdr = null;
try
{
long junkStart = settings.JunkStartOffsets.FirstOrDefault(a => a.Id8 == inStream.Id8)?.Offset ?? 0;
string forceJunkId = settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == inStream.Id8)?.JunkId;
if (forceJunkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", forceJunkId, inStream.Id));
//result.ImageInfo.JunkId = forceJunkId;
inStream.ChangeJunk(forceJunkId);
}
if (junkStart != 0)
{
_log?.LogDetail(string.Format("Junk forced to start at 0x{0}", junkStart.ToString("X8")));
}
hdr = inStream.DiscHeader;
FstFileItem goodFst = rec.GcBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8 && !(inStream.Id8 == "GNHE5d0000" && a.Length == 116));
if (goodFst != null)
{
_log?.LogDetail(string.Format("Recovery: {0}", goodFst.Filename));
}
if (goodFst == null && rec.GcNewBinFiles != null)
{
goodFst = rec.GcNewBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8);
}
if (goodFst != null)
{
goodFst.Populate();
}
ApploaderFileItem goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
if (goodAldr == null && rec.GcNewBinFiles != null)
{
goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcNewBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
}
//is it an action replay (custom hacks)
if ((hdr.ReadUInt32B(0x420) == 0 && inStream.Id8 == "GNHE5d0000") || inStream.Id8 == "DTLX010000" || inStream.Id8 == "102E010007")
{
if (inStream.Id8 == "102E010007")
{
resultMsg = "Aging Disc detected - Skipping recover";
}
else
{
resultMsg = "Datel Action Replay detected - Skipping recover";
}
try
{
using (CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write))
{
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
inStream.Copy(target, pc.OutputSize - hdr.Size);
}
_log?.LogDetail(resultMsg);
}
catch (Exception ex)
{
throw new HandledException(ex, resultMsg);
}
crc.Snapshot("files");
}
else
{
//############################################################################
//# READ DISC START
//########### Header (boot.bin) 0 (read by base stream already)
long srcPos = hdr.Size;
_log?.LogDetail(string.Format("Header Read: {0} - {1}", friendly(inStream.Id), friendly(inStream.Title)));
//########### Header Info (bi2.bin) 0x440
MemorySection bi2bin = MemorySection.Read(inStream, 0x2000);
srcPos += bi2bin.Size;
//########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol
MemorySection appldr = MemorySection.Read(inStream, Math.Min(hdr.ReadUInt32B(0x420) == 0 ? uint.MaxValue : hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - srcPos);
srcPos += appldr.Size;
//########### APP (main.dol)
MemorySection maindol;
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424))
{
maindol = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - srcPos);
}
else
{
maindol = new MemorySection(new byte[0]);
}
srcPos += maindol.Size;
//########### FST (fst.bin)
MemorySection srcFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428));
srcPos += hdr.ReadUInt32B(0x428);
//############################################################################
//# CORRECT ISSUES
List <FstFile> srcFiles = FileSystem.Parse(srcFst, null, inStream.Id, true)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
if (srcFiles == null)
{
throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", hdr.ReadUInt32B(0x424).ToString("X8")));
}
uint crcTmp = Crc.Compute(appldr.Data, 0, Math.Min((int)appldr.Size, 0x20 + (int)(appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18))));
//adjust appldr
if (goodAldr != null && crcTmp != goodAldr.Crc)
{
_log?.LogDetail(string.Format("Replacing appldr.bin crc {0} with Recovery appldr.bin {1}", crcTmp.ToString("X8"), goodAldr.Crc.ToString("X8")));
addedFiles.Add(goodAldr.Filename);
appldr = new MemorySection(File.ReadAllBytes(goodAldr.Filename));
}
//adjust main.dol
if (goodFst != null)
{
if (goodFst.MainDolOffset != hdr.ReadUInt32B(0x420))
{
//main.dol is before fst in image and after in goodfst
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424) && goodFst.MainDolOffset > goodFst.FstOffset)
{
_log?.LogDetail(string.Format("Skipping main.dol at 0x{0} using 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
maindol = new MemorySection(new byte[0]);
}
else
{
_log?.LogDetail(string.Format("Moving main.dol address 0x{0} to 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
}
hdr.WriteUInt32B(0x420, (uint)goodFst.MainDolOffset);
}
if (goodFst.Region != (Region)bi2bin.ReadUInt32B(0x18))
{
_log?.LogDetail(string.Format("Region Changed to {0} from {1}", goodFst.Region.ToString(), ((Region)bi2bin.ReadUInt32B(0x18)).ToString()));
bi2bin.WriteUInt32B(0x18, (uint)goodFst.Region);
}
if (goodFst.MaxFst != hdr.ReadUInt32B(0x42C))
{
_log?.LogDetail(string.Format("Max Fst Size changed to {0} from {1}", goodFst.MaxFst.ToString("X8"), hdr.ReadUInt32B(0x42C).ToString("X8")));
hdr.WriteUInt32B(0x42C, (uint)goodFst.MaxFst);
}
string newTitle = Encoding.ASCII.GetString(goodFst.Title);
if (newTitle != hdr.ReadString(0x20, 0x60 - 0x20))
{
_log?.LogDetail(string.Format("Title changed to '{0}' from '{1}'", newTitle.TrimEnd('\0'), hdr.ReadString(0x20, 0x60 - 0x20).TrimEnd('\0')));
hdr.Write(0x20, goodFst.Title);
}
}
MemorySection fst = srcFst;
List <FstFile> fstFiles = srcFiles;
crcTmp = Crc.Compute(srcFst.Data);
if (goodFst != null && crcTmp != goodFst.Crc)
{
_log?.LogDetail(string.Format("Replacing fst.bin crc {0} with Recovery fst {1}", crcTmp.ToString("X8"), goodFst.Crc.ToString("X8")));
addedFiles.Add(goodFst.Filename);
fst = new MemorySection(goodFst.FstData);
fstFiles = FileSystem.Parse(fst, null, inStream.Id, true).Files.OrderBy(a => a.Offset).ThenBy(a => a.Length).ToList();
}
//adjust fst.bin
if (goodFst != null && (goodFst.FstOffset != hdr.ReadUInt32B(0x424) || goodFst.FstData.Length != hdr.ReadUInt32B(0x428)))
{
_log?.LogDetail(string.Format("Moving / Resizing fst.bin address 0x{0} (Length {1}) to 0x{2} (Length {3})", hdr.ReadUInt32B(0x424).ToString("X8"), hdr.ReadUInt32B(0x428).ToString(), goodFst.FstOffset.ToString("X8"), goodFst.FstData.Length.ToString()));
hdr.WriteUInt32B(0x424, (uint)goodFst.FstOffset);
hdr.WriteUInt32B(0x428, (uint)goodFst.FstData.Length);
}
int c;
int failCount = 0;
int filesMoved = 0;
int nkitJunkFiles = 0;
foreach (FstFile f in srcFiles)
{
FstFile[] fnd = fstFiles.Where(a => a.Name == f.Name && a.Path == f.Path).ToArray();
if (fnd.Length == 0)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: No File Found - {0}/{1} (Length {2})", f.Path, f.Name, f.Length.ToString()));
}
else if (inStream.IsNkit && (c = fnd.Count(a => a.Length != 0 && f.Length == 0)) == 1)
{
nkitJunkFiles++;
}
else if ((c = fnd.Count(a => a.Length == f.Length)) != 1)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: File Size bad - {0}/{1} (Length {2}){3}", f.Path, f.Name, f.Length.ToString(), c <= 1 ? "" : string.Format(" {0} files found", c.ToString())));
}
else if (f.DataOffset != fnd[0].DataOffset)
{
filesMoved++;
}
if (failCount >= 10)
{
break;
}
}
if (failCount != 0)
{
if (failCount >= 10)
{
throw new HandledException(string.Format("{0} or more FST errors found", failCount.ToString()));
}
}
if (filesMoved != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be repositioned when rebuilding this image", filesMoved.ToString(), filesMoved == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (nkitJunkFiles != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be generated from junk when rebuilding this image", nkitJunkFiles.ToString(), nkitJunkFiles == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (goodFst != null)
{
//is the data output so far correct
if (!bruteForceValidHeader(ctx, goodFst, hdr, bi2bin, appldr, maindol, fst, pc))
{
throw new HandledException(string.Format("Post FST Crc Failed 0x{0}", goodFst.PostFstCrc.ToString("X8")));
}
}
//############################################################################
//# WRITE DISC START
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
long dstPos = hdr.Data.Length;
crc.Snapshot("boot.bin");
target.Write(bi2bin.Data, 0, (int)bi2bin.Size);
dstPos += bi2bin.Size;
crc.Snapshot("bi2.bin");
target.Write(appldr.Data, 0, (int)appldr.Size);
dstPos += appldr.Size;
ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos);
dstPos += Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos;
crc.Snapshot("appldr.bin");
target.Write(maindol.Data, 0, (int)maindol.Size);
dstPos += maindol.Size;
if (goodFst != null)
{
long padding = goodFst.FstOffset - dstPos;
ByteStream.Zeros.Copy(target, padding);
dstPos += padding;
}
crc.Snapshot("main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
dstPos += fst.Size;
bool firstFile = true;
Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>();
crc.Snapshot("fst.bin");
//############################################################################
//# WRITE THE FILESYSTEM
int fidx = -1;
FstFile lastFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = fst.Size
};
FstFile nextFile = fstFiles[++fidx];
FstFile cacheFile = null;
long nullsPos = dstPos + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
//########### FILES
foreach (FstFile f in srcFiles) //read the files and write them out as goodFiles (possible order difference)
{
while ((cacheFile = cache.Keys.FirstOrDefault(a => (a.Length == nextFile.Length || (inStream.IsNkit && a.Length == 0)) && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) //write cache
{
bool isJunk = inStream.IsNkit && cacheFile.Length == 0 && cacheFile.Name == nextFile.Name && cacheFile.Path == nextFile.Path;
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, isJunk);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
if (f.DataOffset - srcPos > 0) //skip src junk
{
inStream.Copy(ByteStream.Zeros, f.DataOffset - srcPos);
srcPos += f.DataOffset - srcPos;
}
if (nextFile != null && (f.Length != nextFile.Length || f.Name != nextFile.Name || f.Path != nextFile.Path)) //cache file (nkit junk files are cached)
{
bool isNkitJunk = inStream.IsNkit && f.Length == 0 && nextFile.Length != 0 && f.Name == nextFile.Name && f.Path == nextFile.Path;
byte[] cacheItem = new byte[isNkitJunk ? 8 : f.Length];
inStream.Read(cacheItem, 0, cacheItem.Length); //read the nkit junk data - details (real length and null counts)
cache.Add(f, cacheItem);
srcPos += cacheItem.Length;
}
else //copy file
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, inStream, lastFile, nextFile, patches, inStream, junkStart, false);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
srcPos += f.Length;
}
}
while (nextFile != null && fidx < fstFiles.Count && (cacheFile = cache.Keys.FirstOrDefault(a => a.Length == nextFile.Length && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null)
{
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, false);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
writeDestGap(nullsPos, target, dstPos, pc.OutputSize - dstPos, true, patches, junkStart, inStream);
crc.Snapshot("files");
}
resultMsg = "MatchFail";
uint finalCrc = crc.FullCrc(true);
if (ctx.Dats.RedumpData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Redump";
}
if (ctx.Dats.CustomData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Custom";
}
pc.ReaderCheckPoint2Complete(crc, false, finalCrc, finalCrc, VerifyIsWrite, hdr.Data, resultMsg);
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "RestoreReaderGc.Read - Read and repair"); //don't let the writer lock
}
}
private ExtractResult extractFiles(string id, MemorySection hdr, Stream inStream, Func <ExtractedFile, bool> filter, Action <Stream, ExtractedFile> extract)
{
long mlt = this.IsGameCube ? 1L : 4L;
long srcPos = 0;
List <FstFile> files = new List <FstFile>();
MemorySection bootbin = (this.IsGameCube || this.NStream.IsNkit) ? hdr: MemorySection.Read(inStream, 0x440);
files.Add(new FstFile(null)
{
Name = "boot.bin", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = (int)bootbin.Size, IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += bootbin.Size;
MemorySection bi2bin = MemorySection.Read(inStream, 0x2000);
files.Add(new FstFile(null)
{
Name = "bi2.bin", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = (int)bi2bin.Size, IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += bi2bin.Size;
//########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol
MemorySection appldr = MemorySection.Read(inStream, Math.Min(bootbin.ReadUInt32B(0x420) == 0 ? uint.MaxValue : (bootbin.ReadUInt32B(0x420) * mlt), (bootbin.ReadUInt32B(0x424) * mlt)) - srcPos);
files.Add(new FstFile(null)
{
Name = "appldr.bin", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = 0x20 + appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18), IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += appldr.Size;
//########### APP (main.dol)
MemorySection maindol = null;
if (bootbin.ReadUInt32B(0x420) < bootbin.ReadUInt32B(0x424))
{
maindol = MemorySection.Read(inStream, (bootbin.ReadUInt32B(0x424) * mlt) - srcPos);
uint maindolSize = 0;
for (int i = 0; i < 18; i++)
{
if (maindol.ReadUInt32B(0x0 + (i * 4)) != 0) //7 text offsets, 11 data offsets
{
maindolSize = Math.Max(maindolSize, maindol.ReadUInt32B(0x0 + (i * 4)) + maindol.ReadUInt32B(0x90 + (i * 4)));
}
}
files.Add(new FstFile(null)
{
Name = "main.dol", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = maindolSize, IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += maindol.Size;
}
MemorySection fstbin = MemorySection.Read(inStream, bootbin.ReadUInt32B(0x428) * mlt);
files.Add(new FstFile(null)
{
Name = "fst.bin", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = (int)fstbin.Size, IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += fstbin.Size;
List <FstFile> fstFiles = FileSystem.Parse(fstbin, null, id, this.IsGameCube)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
if (fstFiles == null)
{
throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", (bootbin.ReadUInt32B(0x424) * mlt).ToString("X8")));
}
files.AddRange(fstFiles);
Dictionary <string, MemorySection> mem = new Dictionary <string, MemorySection>();
mem.Add("boot.bin", bootbin);
mem.Add("bi2.bin", bi2bin);
mem.Add("appldr.bin", appldr);
mem.Add("main.dol", maindol);
mem.Add("fst.bin", fstbin);
List <ExtractedFile> exfiles = files.OrderBy(a => a.Offset).ThenBy(a => a.Length).Select(a => new ExtractedFile(this.IsGameCube ? DiscType.GameCube : DiscType.Wii,
this.NStream.Id8,
this.IsGameCube ? null : id,
a.DataOffset,
a.Length,
a.Path,
a.Name,
mem.ContainsKey(a.Name) ? ExtractedFileType.System : ExtractedFileType.File)).Where(a => filter(a)).ToList();
if (files == null || files.Count == 0)
{
return(createExtractResult((Region)bi2bin.ReadUInt32B(0x18), null));
}
foreach (ExtractedFile f in exfiles) //read the files and write them out as goodFiles (possible order difference)
{
if (srcPos < f.Offset)
{
inStream.Copy(Stream.Null, f.Offset - srcPos);
srcPos += f.Offset - srcPos;
}
if (f.Type == ExtractedFileType.System)
{
using (MemoryStream ms = new MemoryStream(mem[f.Name].Data))
extract(ms, f);
}
else
{
extract(inStream, f);
srcPos += f.Length;
}
}
return(createExtractResult((Region)bi2bin.Read8(0x18), null));
}
private long writeFile(ref long nullsPos, Stream dest, long dstPos, ref bool firstFile, Stream srcStream, FstFile lastFile, FstFile file, List <JunkRedumpPatch> patches, NStream nstream, long junkStart, bool isJunkFile)
{
//Debug.WriteLine(string.Format(@"{0} : {1} : {2} : {3}/{4}", file.DataOffset.ToString("X8"), (file.DataOffset + file.Length).ToString("X8"), /*(nextFile.DataOffset - lastEnd).ToString("X8"),*/ file.Length.ToString("X8"), file.Path, file.Name));
//file found
long gap = writeDestGap(nullsPos, dest, dstPos, file.DataOffset - dstPos, firstFile, patches, junkStart, nstream);
firstFile = false;
bool missing = false;
if (file.Length == 0)
{
// missing = true;
}
else if (isJunkFile)
{
missing = true;
MemorySection ms = MemorySection.Read(srcStream, 8);
long size = ms.ReadUInt32B(0);
GapType gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
//set nullsPos value if zerobyte file without junk
if (gt == GapType.JunkFile)
{
nullsPos = Math.Min(nullsPos - (dstPos + gap), 0); //reset the nulls
long nulls = (size & 0xFC) >> 2;
long junkFileLen = ms.ReadUInt32B(4);
if (junkFileLen != file.Length)
{
throw new HandledException(string.Format("NKit Junk file restoration length mismatch {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
ByteStream.Zeros.Copy(dest, nulls);
nstream.JunkStream.Position = (dstPos + gap) + nulls;
nstream.JunkStream.Copy(dest, junkFileLen - nulls);
_log?.LogDetail(string.Format("Generated file content with Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
else
{
throw new HandledException(string.Format("NKit Junk file restoration bytes invalid {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
}
else
{
byte[] f = new byte[Math.Min(0x30, file.Length)];
srcStream.Read(f, 0, f.Length); //then read while junk is created
if (lastFile.DataOffset == file.DataOffset && lastFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size
{
nullsPos = dstPos + 0x1CL; //will already be aligned
}
int nulls = (int)(nullsPos - (dstPos + gap));
nstream.JunkStream.Position = file.DataOffset; //async junk gen
int countNulls = 0;
for (int i = 0; i < f.Length && f[i] == 0; i++)
{
countNulls++;
}
if (f.Length > nulls && countNulls < f.Length) //don't test all nulls
{
missing = nstream.JunkStream.Compare(f, 0, f.Length, Math.Max(0, nulls)) == f.Length;
}
if (missing)
{
_log?.LogDetail(string.Format("File content is Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
dest.Write(f, 0, f.Length);
srcStream.Copy(dest, file.Length - f.Length); //copy file
}
if (!missing) //reset the gap when no junk
{
nullsPos = dstPos + gap + file.Length + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
}
return(gap + file.Length);
}
public ExtractResult ExtractRecoveryFilesGc()
{
MemorySection hdr = NStream.DiscHeader;
MemorySection bi2bin;
List <ExtractRecoveryResult> result = new List <ExtractRecoveryResult>();
NCrc crc = new NCrc();
uint appLdrCrc = 0;
MemorySection fst = null;
uint fstCrc = 0;
string fn;
string tmpFullName;
Log?.Log(string.Format("Processing: {0}", SourceFileName));
int storeType;
using (CryptoStream bw = new CryptoStream(ByteStream.Zeros, crc, CryptoStreamMode.Write))
{
bw.Write(hdr.Data, 0, (int)hdr.Size);
bi2bin = MemorySection.Read(NStream, 0x2000);
bw.Write(bi2bin.Data, 0, (int)bi2bin.Size);
MemorySection aplHdr = MemorySection.Read(NStream, 0x20);
byte[] appLdr = new byte[0x20 + aplHdr.ReadUInt32B(0x14) + aplHdr.ReadUInt32B(0x18)];
Array.Copy(aplHdr.Data, appLdr, (int)aplHdr.Size);
NStream.Read(appLdr, (int)aplHdr.Size, appLdr.Length - (int)aplHdr.Size);
appLdrCrc = Crc.Compute(appLdr);
fn = string.Format("appldr[{0}][{1}].bin", aplHdr.ReadString(0, 10).Replace("/", ""), appLdrCrc.ToString("X8"));
Log?.Log(string.Format(" 1 of 2 - Extracting appldr.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + appLdrCrc.ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, appLdr);
if ((storeType = storeRecoveryFile(this.Settings.RedumpAppldrCrcs.Contains(appLdrCrc), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
bw.Write(appLdr, 0, appLdr.Length); //add to fullcrc
NStream.Copy(bw, hdr.ReadUInt32B(0x424) - (0x2440 + appLdr.Length));
byte[] fstData = new byte[(int)((4 * 4) + (0x60 - 0x20) + hdr.ReadUInt32B(0x428))];
fst = new MemorySection(fstData); //info: dolAddr, fstAddr, maxfst, region, title, fst
NStream.Read(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
fst.WriteUInt32B(0x00, hdr.ReadUInt32B(0x420)); //dol
fst.WriteUInt32B(0x04, hdr.ReadUInt32B(0x424)); //fstAddr
fst.WriteUInt32B(0x08, hdr.ReadUInt32B(0x42C)); //maxfst
fst.WriteUInt32B(0x0c, bi2bin.ReadUInt32B(0x18)); //region
fst.Write(0x10, hdr.Read(0x20, 0x60 - 0x20)); //title
fstCrc = Crc.Compute(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428)); //fst
bw.Write(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
crc.Snapshot("crc");
}
//fst checksums are postfst
fn = string.Format("fst[{0}][{1}][{2}][{3}].bin", NStream.Id8, appLdrCrc.ToString("X8"), fstCrc.ToString("X8"), crc.FullCrc().ToString("X8"));
Log?.Log(string.Format(" 2 of 2 - Extracting fst.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + crc.FullCrc().ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, fst.Data);
if ((storeType = storeRecoveryFile(this.Settings.RedumpFstCrcs.Contains(crc.FullCrc()), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
return(createExtractResult((Region)bi2bin.Read8(0x18), result.ToArray()));
}
