private long partitionWrite(Stream inStream, NCrc crc, Stream target, WiiPartitionSection pHdr, Context ctx, Coordinator pc, NkitInfo imageInfo, ScrubManager scrub, WiiHashStore hashes, long fstFileAlignment)
{
#if DEHASH
return(pHdr.NewPartitionDataLength = inStream.Copy(target, pHdr.PartitionDataLength, null));
#elif DECRYPT
inStream.Copy(target, pHdr.PartitionLength, null);
return(pHdr.NewPartitionDataLength = pHdr.PartitionDataLength);
#endif
MemorySection hdr = MemorySection.Read(inStream, 0x440);
//ProgressResult result = ctx.Result;
long mlt = 4L; //GC 1L
long logicalSize = pHdr.PartitionDataLength; //GC: ctx.ImageLength;
long physicalSize = pHdr.PartitionDataLength; //GC: result.ImageInfo.IsoSize;
string junkId = hdr.ReadString(0, 4);
//bool write = true;
List <string> addedFiles = new List <string>();
long srcPos;
long dstPos = 0;
JunkStream js = new JunkStream(junkId, hdr.Read8(6), logicalSize);
try
{
if (junkId == "\0\0\0\0")
{
srcPos = hdr.Size;
imageInfo.BytesData = srcPos;
_log?.LogDetail("Null Partition ID, preserving partition as raw");
ConvertFile cf = new ConvertFile(logicalSize - hdr.Size, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.Size, Offset = hdr.Size, Length = 0
},
};
long nullsPos = 0;
target.Write(hdr.Data, 0, (int)hdr.Size); //0x400
target.Write(pHdr.Header.Data, 0x2bc, 4); //copy the original partition length
dstPos += 0x440 + 4 + NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
hdr.WriteString(0x200, 8, "NKIT v01");
hdr.WriteUInt32B(0x210, (uint)(pHdr.PartitionLength / mlt));
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
crc.Snapshot(junkId + " PrtHdr");
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot(junkId + " Fst");
target.Write(hashes.FlagsToByteArray(), 0, (int)hashes.FlagsLength);
crc.Snapshot(junkId + " HashFlags");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos + hashes.FlagsLength;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, physicalSize, hdr, fst, false, fstFileAlignment, out error);
imageInfo.BytesData = srcPos;
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(pHdr.PartitionDataLength - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
List <ConvertFile> missing;
NkitFormat.NkitWriteFileSystem(ctx, imageInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out missing, scrub, pHdr.PartitionDataLength, _log);
dstPos += hashes.HashesToStream(target);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
crc.Crcs[crc.Crcs.Length - 1].PatchData = hashes.FlagsToByteArray();
crc.Crcs[crc.Crcs.Length - 1].PatchCrc = Crc.Compute(crc.Crcs[crc.Crcs.Length - 2].PatchData);
crc.Crcs[crc.Crcs.Length - 2].PatchData = fst.Data;
crc.Crcs[crc.Crcs.Length - 2].PatchCrc = Crc.Compute(fst.Data);
}
}
}
catch (Exception ex)
{
throw new HandledException(ex, "NkitWriterGc.Write - Convert");
}
return(dstPos);
}
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
{
try
{
WiiDiscHeaderSection hdr = null;
WiiPartitionHeaderSection pHdr = null;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
NCrc crc = new NCrc();
Crc updateCrc = new Crc();
bool updateRemoved = false;
string updateTmpFileName = null;
string updateFileName = null;
bool extractingUpdate = false;
CryptoStream updateCrcStream = null;
NStream updateTarget = null;
CryptoStream target = null;
MemorySection removedUpdateFiller = null;
int preservedHashCount = 0;
NkitInfo nkitDiscInfo = new NkitInfo();
long fstFileAlignment = -1;
WiiPartitionSection lastPart = null;
long dstPos = 0;
long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength
string ignoreJunkId;
pc.WriterCheckPoint1WriteReady(out ignoreJunkId); //wait until read has written the header and set the length
NDisc disc = new NDisc(_log, inStream);
foreach (IWiiDiscSection s in disc.EnumerateSections(imageSize)) //no fixing, image should be good //ctx.ImageLength
{
if (s is WiiDiscHeaderSection)
{
hdr = (WiiDiscHeaderSection)s;
hdr.Write8(0x60, 1);
hdr.Write8(0x61, 1);
fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == hdr.Id8)?.Item2 ?? -1;
target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(hdr.Data, 0, hdr.Data.Length); //write the header
nkitDiscInfo.BytesData += hdr.Size;
dstPos += hdr.Size;
crc.Snapshot("Disc Header");
}
else if (s is WiiPartitionSection)
{
WiiPartitionSection ps = (WiiPartitionSection)s;
WiiHashStore hashes = new WiiHashStore(ps.PartitionDataLength);
ScrubManager scrub = ps.Header.ScrubManager;
NkitInfo nkitPartInfo = new NkitInfo();
if (ps.Header.Type == PartitionType.Update && ctx.Settings.NkitUpdatePartitionRemoval && hdr.Partitions.Count() > 1) //only remove update if there's more partitions
{
updateRemoved = true;
extractingUpdate = true;
pHdr = ps.Header;
extractingUpdate = true;
updateCrcStream = new CryptoStream(ByteStream.Zeros, updateCrc, CryptoStreamMode.Write);
crc.Initialize();
nkitPartInfo.BytesData += disc.WriteRecoveryPartitionData(updateCrcStream, false, ps, 0, out updateTmpFileName, out updateFileName, out updateTarget);
_log?.LogDetail(string.Format("Extracted and Removed {0} Recovery Partition: [{1}]", ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));
removedUpdateFiller = new MemorySection(new byte[0x8000]);
removedUpdateFiller.Write(0, hdr.Read(0x40000, 0x100)); //backup the original partition table in case it's non standard in some way
}
else
{
target.Write(ps.Header.Data, 0, (int)ps.Header.Size);
nkitDiscInfo.BytesData += ps.Header.Size;
crc.Snapshot(ps.Id + " Hdr");
crc.Crcs.Last().PatchData = ps.Header.Data;
dstPos += ps.Header.Size;
ps.NewDiscOffset = dstPos;
//long written = 0;
using (StreamCircularBuffer decrypted = new StreamCircularBuffer(ps.PartitionDataLength, null, null, output =>
{
//read decrypted partition
foreach (WiiPartitionGroupSection pg in ps.Sections)
{
nkitPartInfo.BytesHashesData += (pg.Size / 0x8000 * 0x400); //size of input hashes
if (pg.PreserveHashes())
{
nkitPartInfo.BytesHashesPreservation += hashes.Preserve(pg.Offset, pg.Decrypted, pg.Size);
if (++preservedHashCount >= 1500) //too many will bomb the patch caching when converting back. Something is wrong
{
throw pc.SetWriterException(new HandledException("Over 1500 hashes preserved, aborting as image is corrupt or poorly scrubbed."));
}
}
#if !DECRYPT
for (int i = 0; i < pg.Size / 0x8000; i++)
{
output.Write(pg.Decrypted, (i * 0x8000) + 0x400, 0x7c00);
}
#else
output.Write(pg.Decrypted, 0, (int)pg.Size);
#endif
}
}))
{
long len = partitionWrite(decrypted, crc, target, ps, ctx, pc, nkitPartInfo, scrub, hashes, fstFileAlignment);
ps.NewPartitionDataLength = len;
dstPos += len;
lastPart = ps;
}
}
NkitFormat.LogNkitInfo(nkitPartInfo, _log, ps.Id, false);
lastPartitionId = ps.Id;
lastPartitionType = ps.Header.Type;
}
else if (s is WiiFillerSection)
{
WiiFillerSection fs = (WiiFillerSection)s;
ScrubManager scrub = new ScrubManager(null);
JunkStream junk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
if (lastPartitionType == PartitionType.Update && updateRemoved)
{
//preserve the original partition table and update filename
target.Write(removedUpdateFiller.Data, 0, (int)removedUpdateFiller.Size); //remove update partition by adding a 32k placeholder. Avoid having a non update partition at 0x50000
nkitDiscInfo.BytesPreservationDiscPadding += removedUpdateFiller.Size;
nkitDiscInfo.BytesGaps += fs.Size;
dstPos += removedUpdateFiller.Size;
crc.Snapshot(string.Format("{0}Replacement Filler", string.IsNullOrEmpty(lastPartitionId) ? "" : (" " + SourceFiles.CleanseFileName(lastPartitionId).PadRight(4))));
if (extractingUpdate)
{
int storeType;
if ((storeType = disc.WriteRecoveryPartitionFiller(updateCrcStream, junk, fs.DiscOffset, true, true, fs, updateTarget, updateTmpFileName, ref updateFileName, updateCrc, true)) != 0)
{
_log.LogDetail(string.Format("{0}Update recovery partition stored: {1}", storeType == 2 ? "Other " : "", updateFileName));
}
extractingUpdate = false;
}
}
else
{
if (fs.Size != 0)
{
using (StreamCircularBuffer filler = new StreamCircularBuffer(fs.Size, null, null, output =>
{
foreach (WiiFillerSectionItem item in ((WiiFillerSection)s).Sections)
{
output.Write(item.Data, 0, (int)item.Size);
}
}))
{
Gap gap = new Gap(fs.Size, false);
long srcPos = fs.DiscOffset;
long gapLen = gap.Encode(filler, ref srcPos, lastPartitionType == PartitionType.Update ? fs.Size : 0x1cL, fs.Size, junk, scrub, target, _log);
nkitDiscInfo.BytesPreservationData += gapLen;
nkitDiscInfo.BytesGaps += fs.Size;
dstPos += gapLen;
if (lastPart != null)
{
lastPart.NewPartitionDataLength += gapLen;
}
}
}
//pad partition to 32k
int gapLen2 = (int)(dstPos % 0x8000 == 0 ? 0 : 0x8000 - (dstPos % 0x8000));
ByteStream.Zeros.Copy(target, gapLen2);
nkitDiscInfo.BytesPreservationDiscPadding += gapLen2;
dstPos += gapLen2;
if (lastPart != null)
{
lastPart.NewPartitionDataLength += gapLen2;
}
if (lastPart != null)
{
lastPart.Header.WriteUInt32B(0x2bc, (uint)(lastPart.NewPartitionDataLength / 4)); //updates the array in the crc data
hdr.Partitions.First(a => a.DiscOffset == lastPart.DiscOffset).DiscOffset = (lastPart.NewDiscOffset - 0x20000);
}
crc.Snapshot(string.Format("{0}{1}Files+Filler", lastPartitionId ?? "", string.IsNullOrEmpty(lastPartitionId) ? "" : " "));
}
}
}
NkitFormat.LogNkitInfo(nkitDiscInfo, _log, hdr.Id, true);
foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
{
ci.PatchCrc = Crc.Compute(ci.PatchData);
}
NCrc readerCrcs;
uint validationCrc;
pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.
if (updateRemoved && crc.Crcs.Length > 2) //freeloader wii only has update partition
{
hdr.RemoveUpdatePartition(crc.Crcs[2].Offset);
}
hdr.UpdateOffsets();
hdr.WriteString(0x200, 8, "NKIT v01"); //header and version
hdr.Write8(0x60, 1);
hdr.Write8(0x61, 1);
hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
hdr.WriteUInt32B(0x210, (uint)(imageSize / 4L)); //ctx.ImageLength
hdr.WriteUInt32B(0x218, updateCrc.Value); //Update crc - Only if removed
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); //update with magic applied
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "NKit Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "NkitWriterWii.Write - Convert");
}
}
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
{
try
{
long mlt = 1L; //for Wii: 4L
long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength
pc.WriterCheckPoint1WriteReady(out string junkId); //wait until read has written the header and set the length
List <string> addedFiles = new List <string>();
NCrc crc = new NCrc();
long srcPos;
long dstPos = 0;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
string id8 = string.Concat(hdr.ReadString(0, 6), hdr.Data[6].ToString("X2"), hdr.Data[7].ToString("X2"));
if (junkId == null)
{
junkId = ctx.Settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == id8)?.JunkId;
if (junkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", junkId, id8.Substring(0, 4)));
}
}
if (junkId == null)
{
junkId = hdr.ReadString(0, 4);
}
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
long fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == id8)?.Item2 ?? -1;
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
crc.Snapshot("hdr.bin");
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot("fst.bin");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
JunkStream js = new JunkStream(junkId, hdr.Read8(6), NStream.FullSizeGameCube);
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, imageSize, hdr, fst, true, fstFileAlignment, out string error); //Size isn't important for writing //result.ImageInfo.IsoSize
NkitInfo nkitInfo = new NkitInfo
{
BytesData = srcPos,
BytesGaps = 0,
BytesJunkFiles = 0,
BytesPreservationData = 0,
BytesPreservationDiscPadding = 0
};
ScrubManager scrub = new ScrubManager();
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
NkitFormat.NkitWriteFileSystem(ctx, nkitInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out List <ConvertFile> missing, scrub, imageSize, _log);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
}
if (dstPos % 0x800 != 0)
{
long l = 0x800 - (dstPos % 0x800);
ByteStream.Zeros.Copy(target, l);
dstPos += l;
nkitInfo.BytesPreservationDiscPadding += l;
}
crc.Snapshot("files");
NkitFormat.LogNkitInfo(nkitInfo, _log, hdr.ReadString(0, 4), true);
pc.WriterCheckPoint2Complete(out NCrc readerCrcs, out uint validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.
hdr.WriteUInt32B(0x420, (uint)mainDolAddr);
hdr.WriteString(0x200, 8, "NKIT v01"); //header and version
hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
hdr.WriteUInt32B(0x210, (uint)imageSize); //result.ImageInfo.IsoSize);
hdr.WriteString(0x214, 4, hdr.ReadString(0, 4) != junkId ? junkId : "\0\0\0\0");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, "NKit Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "NkitWriterGc.Write - Convert");
}
}
public void Write(Context ctx, Stream input, Stream output, Coordinator pc)
{
try
{
long imageSize = pc.OutputSize;
string junkId;
pc.WriterCheckPoint1WriteReady(out junkId); //wait until read has written the header and set the length
MemorySection ms = new MemorySection(new byte[4 + 4 + 8 + 8 + 4 + 4]);
int blockSize = 0x4000;
int blocks = (int)(imageSize / blockSize) + (imageSize % blockSize == 0 ? 0 : 1);
ms.WriteUInt32L(0x00, 0xB10BC001);
ms.WriteUInt32L(0x04, 0); //insert NKIT later
ms.WriteUInt64L(0x08, 0); //size place holder
ms.WriteUInt64L(0x10, (ulong)imageSize);
ms.WriteUInt32L(0x18, (uint)blockSize);
ms.WriteUInt32L(0x1C, (uint)blocks);
MemorySection pnt = new MemorySection(new byte[blocks * 8]);
MemorySection hsh = new MemorySection(new byte[blocks * 4]);
long offset = 0;
NCrc crc = new NCrc();
CryptoStream target = new CryptoStream(output, crc, CryptoStreamMode.Write);
target.Write(ms.Data, 0, (int)ms.Size);
crc.Snapshot("Header");
target.Write(pnt.Data, 0, (int)pnt.Size);
crc.Snapshot("Pointers");
target.Write(hsh.Data, 0, (int)hsh.Size);
crc.Snapshot("Hashes");
long dataOffset = (pnt.Size + hsh.Size + ms.Size); //fso.position
int rdBlk = 0;
int wrBlk = 0;
object rdBlkLock = new object();
object wrBlkLock = new object();
Task[] tasks = new Task[3];
ManualResetEvent mr = new ManualResetEvent(false);
for (int i = 0; i < tasks.Length; i++) //4 threads
{
tasks[i] = (new TaskFactory()).StartNew((Object prm) =>
{
ManualResetEvent m = (ManualResetEvent)prm;
int blkIdx;
byte[] rawBlk = new byte[blockSize];
byte[] buff = new byte[blockSize + 0x100];
using (MemoryStream blk = new MemoryStream(buff))
{
while (true)
{
lock (rdBlkLock)
{
blkIdx = rdBlk++;
if (blkIdx >= blocks)
{
return; //no more processing
}
int read = input.Read(rawBlk, 0, (int)Math.Min(imageSize - (blkIdx * blockSize), blockSize));
if (read != blockSize)
{
Array.Clear(rawBlk, read, blockSize - read);
}
}
blk.Position = 0;
ZlibStream zl = new ZlibStream(blk, SharpCompress.Compressors.CompressionMode.Compress, SharpCompress.Compressors.Deflate.CompressionLevel.BestCompression, Encoding.Default);
zl.FlushMode = FlushType.Finish;
zl.Write(rawBlk, 0, blockSize);
zl.Flush();
while (blkIdx != wrBlk)
{
m.WaitOne();
}
mr.Reset();
if (blk.Position < blockSize)
{
target.Write(buff, 0, (int)blk.Position);
pnt.WriteUInt64L(blkIdx * 8, (ulong)(offset));
hsh.WriteUInt32L(blkIdx * 4, adler32(buff, (int)blk.Position));
offset += (int)blk.Position;
}
else
{
target.Write(rawBlk, 0, blockSize);
pnt.WriteUInt64L(blkIdx * 8, (ulong)(offset) | 0x8000000000000000);
hsh.WriteUInt32L(blkIdx * 4, adler32(rawBlk, blockSize));
offset += (int)blockSize;
}
target.Flush();
wrBlk++;
mr.Set();
}
}
}, mr);
}
Task.WaitAll(tasks);
crc.Snapshot("Files");
NCrc readerCrcs;
uint validationCrc;
pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, null, dataOffset + offset); //wait until reader has completed and get crc patches.
ms.WriteUInt64L(0x08, (ulong)offset);
crc.Crcs[0].PatchCrc = Crc.Compute(ms.Data);
crc.Crcs[0].PatchData = ms.Data;
crc.Crcs[1].PatchCrc = Crc.Compute(pnt.Data);
crc.Crcs[1].PatchData = pnt.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(hsh.Data);
crc.Crcs[2].PatchData = hsh.Data;
ms.WriteUInt32B(0x04, CrcForce.Calculate(crc.FullCrc(true), output.Length, readerCrcs.FullCrc(true), 0x04, ms.ReadUInt32B(0x04))); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(ms.Data);
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "GCZ Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "GczWriter.Write - Compress");
}
}
/// <summary>
/// Extracts files from partitions, this is not random access. The iso is read in it's entirety
/// </summary>
public ExtractResult ExtractRecoveryFilesWii()
{
List <ExtractRecoveryResult> result = new List <ExtractRecoveryResult>();
List <WiiPartitionInfo> toExtract = new List <WiiPartitionInfo>();
WiiDiscHeaderSection hdr = null;
WiiPartitionHeaderSection pHdr = null;
NStream target = null;
bool extracting = false;
int channel = 1;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
string fileName = null;
string tmpFileName = null;
int extracted = 0;
Crc crc = new Crc();
Stream crcStream = null;
long imageSize = this.NStream.RecoverySize; //for Wii: pHdr.PartitionDataLength
bool isIso = false; //force to always scrub //Path.GetExtension(_name).ToLower() == ".iso";
foreach (IWiiDiscSection s in this.EnumerateSectionsFix(false, true, false))
{
if (s is WiiDiscHeaderSection)
{
hdr = (WiiDiscHeaderSection)s;
Log?.Log(string.Format("Processing: {0}", SourceFileName));
toExtract = hdr.Partitions.Where(a => a.Type != PartitionType.Data && a.Type != PartitionType.GameData).ToList();
if (toExtract.Count() == 0)
{
Log?.Log(string.Format(" Skipped: No Recovery Partitions to extract - {0}", SourceFileName));
break;
}
}
else if (s is WiiPartitionSection)
{
WiiPartitionSection ps = (WiiPartitionSection)s;
if (!toExtract.Any(a => a.DiscOffset == ps.DiscOffset))
{
Log?.Log(string.Format(" Skipping {0} Partition: [{1}]...", ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));
continue;
}
extracting = true;
pHdr = ps.Header;
extracting = true;
Log?.Log(string.Format(" {0} of {1} - Extracting {2} Recovery Partition: [{3}]", (extracted + 1).ToString(), toExtract.Count().ToString(), ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));
crcStream = new CryptoStream(ByteStream.Zeros, crc, CryptoStreamMode.Write);
crc.Initialize();
WriteRecoveryPartitionData(crcStream, isIso, ps, channel, out tmpFileName, out fileName, out target);
lastPartitionId = ps.Id;
lastPartitionType = ps.Header.Type;
}
else if (s is WiiFillerSection)
{
JunkStream junk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
WiiFillerSection fs = (WiiFillerSection)s;
if (extracting)
{
int storeType;
if ((storeType = WriteRecoveryPartitionFiller(crcStream, junk, fs.DiscOffset, pHdr.Type == PartitionType.Update, false, fs, target, tmpFileName, ref fileName, crc, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fileName, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = false
});
}
if (pHdr.Type != PartitionType.Update)
{
channel++;
}
extracted++;
extracting = false;
bool complete = (extracted == toExtract.Count());
if (complete)
{
break;
}
}
}
}
return(createExtractResult((Region)hdr.ReadUInt32B(0x4e000), result.ToArray()));
}
internal int WriteRecoveryPartitionFiller(Stream crcStream, JunkStream junk, long pos, bool isUpdate, bool isNkit, WiiFillerSection fs, NStream target, string tmpFileName, ref string fileName, Crc crc, bool logAsDetail)
{
long nullBlocks = 0; //check for random junk - only on a handful of launch releases (Rampage, Ant Bully, Grim Adventures, Happy Feet etc)
junk.Position = pos;
long leadingNullsPos = pos;
foreach (WiiFillerSectionItem fi in fs.Sections)
{
crcStream.Write(fi.Data, 0, (int)fi.Size);
for (int i = 0; i < fi.Size; i += 0x8000)
{
int len = (int)Math.Min(0x8000L, fi.Size - (long)i);
bool match = junk.Compare(fi.Data, i, len, junk.Position == leadingNullsPos ? 0x1c : 0) == len;
if (match)
{
nullBlocks++;
}
else
{
if (nullBlocks != 0)
{
if (pos == leadingNullsPos)
{
ByteStream.Zeros.Copy(target, 0x1cL);
junk.Position = pos + 0x1cL;
junk.Copy(target, (nullBlocks * 0x8000L) - 0x1cL);
}
else
{
junk.Position = pos;
junk.Copy(target, nullBlocks * 0x8000L);
}
pos += nullBlocks * 0x8000L;
}
nullBlocks = 0;
target.Write(fi.Data, i, len);
}
}
}
target.Close();
crcStream.Close();
string fName = (fileName += crc.Value.ToString("X8"));
bool redump;
if (isUpdate)
{
redump = this.Settings.RedumpUpdateCrcs.Contains(crc.Value);
}
else
{
redump = this.Settings.RedumpChannels.FirstOrDefault(a => fName.StartsWith(a.Item1)) != null;
}
if (isNkit)
{
Directory.CreateDirectory(Settings.NkitRecoveryFilesPath);
}
return(storeRecoveryFile(redump, tmpFileName, fileName, Settings.RecoveryFilesPath, isNkit ? Settings.NkitRecoveryFilesPath : Settings.OtherRecoveryFilesPath, logAsDetail));
//rename the file, delete if dupe
}
public void SaveMainDol(string outPath)
{
MemorySection hdr = this.NStream.DiscHeader;
this.NStream.Copy(ByteStream.Zeros, hdr.ReadUInt32B(0x420) - 0x440);
//dol info https://github.com/jchv/gcmtools/blob/master/gcm/dol.hpp
byte[] dolLen = new byte[4];
this.NStream.Read(dolLen, 0, 4);
MemorySection mx = new MemorySection(dolLen);
byte[] dolHdr = new byte[mx.ReadUInt32B(0)];
dolLen.CopyTo(dolHdr, 0);
this.NStream.Read(dolHdr, 4, dolHdr.Length - 4);
MemorySection maindol = new MemorySection(dolHdr);
uint maindolSize = 0;
for (int i = 0; i < 18; i++)
{
if (maindol.ReadUInt32B(0x0 + (i * 4)) != 0) //7 text offsets, 11 data offsets
{
maindolSize = Math.Max(maindolSize, maindol.ReadUInt32B(0x0 + (i * 4)) + maindol.ReadUInt32B(0x90 + (i * 4)));
}
}
byte[] dol = new byte[maindol.Size + maindolSize];
dolHdr.CopyTo(dol, 0);
this.NStream.Read(dol, dolHdr.Length, dol.Length - dolHdr.Length);
string dolName = Path.Combine(outPath, string.Format("main[{0}][{1}][{2}].dol", this.NStream.Id8, hdr.ReadUInt32B(0x420).ToString("X8"), Crc.Compute(dol).ToString("X8")));
if (!File.Exists(dolName))
{
File.WriteAllBytes(dolName, dol);
}
}
public ExtractResult ExtractRecoveryFilesGc()
{
MemorySection hdr = NStream.DiscHeader;
MemorySection bi2bin;
List <ExtractRecoveryResult> result = new List <ExtractRecoveryResult>();
NCrc crc = new NCrc();
uint appLdrCrc = 0;
MemorySection fst = null;
uint fstCrc = 0;
string fn;
string tmpFullName;
Log?.Log(string.Format("Processing: {0}", SourceFileName));
int storeType;
using (CryptoStream bw = new CryptoStream(ByteStream.Zeros, crc, CryptoStreamMode.Write))
{
bw.Write(hdr.Data, 0, (int)hdr.Size);
bi2bin = MemorySection.Read(NStream, 0x2000);
bw.Write(bi2bin.Data, 0, (int)bi2bin.Size);
MemorySection aplHdr = MemorySection.Read(NStream, 0x20);
byte[] appLdr = new byte[0x20 + aplHdr.ReadUInt32B(0x14) + aplHdr.ReadUInt32B(0x18)];
Array.Copy(aplHdr.Data, appLdr, (int)aplHdr.Size);
NStream.Read(appLdr, (int)aplHdr.Size, appLdr.Length - (int)aplHdr.Size);
appLdrCrc = Crc.Compute(appLdr);
fn = string.Format("appldr[{0}][{1}].bin", aplHdr.ReadString(0, 10).Replace("/", ""), appLdrCrc.ToString("X8"));
Log?.Log(string.Format(" 1 of 2 - Extracting appldr.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + appLdrCrc.ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, appLdr);
if ((storeType = storeRecoveryFile(this.Settings.RedumpAppldrCrcs.Contains(appLdrCrc), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
bw.Write(appLdr, 0, appLdr.Length); //add to fullcrc
NStream.Copy(bw, hdr.ReadUInt32B(0x424) - (0x2440 + appLdr.Length));
byte[] fstData = new byte[(int)((4 * 4) + (0x60 - 0x20) + hdr.ReadUInt32B(0x428))];
fst = new MemorySection(fstData); //info: dolAddr, fstAddr, maxfst, region, title, fst
NStream.Read(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
fst.WriteUInt32B(0x00, hdr.ReadUInt32B(0x420)); //dol
fst.WriteUInt32B(0x04, hdr.ReadUInt32B(0x424)); //fstAddr
fst.WriteUInt32B(0x08, hdr.ReadUInt32B(0x42C)); //maxfst
fst.WriteUInt32B(0x0c, bi2bin.ReadUInt32B(0x18)); //region
fst.Write(0x10, hdr.Read(0x20, 0x60 - 0x20)); //title
fstCrc = Crc.Compute(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428)); //fst
bw.Write(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
crc.Snapshot("crc");
}
//fst checksums are postfst
fn = string.Format("fst[{0}][{1}][{2}][{3}].bin", NStream.Id8, appLdrCrc.ToString("X8"), fstCrc.ToString("X8"), crc.FullCrc().ToString("X8"));
Log?.Log(string.Format(" 2 of 2 - Extracting fst.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + crc.FullCrc().ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, fst.Data);
if ((storeType = storeRecoveryFile(this.Settings.RedumpFstCrcs.Contains(crc.FullCrc()), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
return(createExtractResult((Region)bi2bin.Read8(0x18), result.ToArray()));
}
public HeaderBruteForcer(uint[] updateCrcs, SortedList <uint, bool> checkCrcs, Tuple <byte[], int[]>[] regionData, params byte[][] headers)
{
_updateCrcs = updateCrcs;
_checkCrcs = checkCrcs;
//DateTime dt = DateTime.Now;
_hdrs = headers.Select(a => new headerCrc()
{
Crc = Crc.Compute(a), CrcPreRegion = Crc.Compute(a, 0, 0x4e000), CrcRegion = Crc.Compute(a, 0x4e000, 0x20), CrcPostRegion = Crc.Compute(a, 0x4e020, 0x1fe0), Data = a
}).ToList();
MemorySection ms = new MemorySection(headers[0]);
_origRegion = (int)ms.ReadUInt32B(0x4e000);
_origRegionData = ms.Read(0x4e010, 0x10);
_origHeaderCrcs = _hdrs.Select(a => a.Crc).ToArray(); //original header Crcs
for (int h = _hdrs.Count - 1; h >= 1; h--)
{
if (_hdrs[h].Crc == _hdrs[h - 1].Crc)
{
_hdrs.RemoveAt(h); //remove duplicate headers
}
}
_regionData = new List <regionCrc>();
for (int h = 0; h < _hdrs.Count; h++)
{
for (int i = 0; i < regionData.Length; i++)
{
for (int r = 0; r < regionData[i].Item2.Length; r++)
{
_regionData.Add(new regionCrc()
{
Crc = 0, Header = _hdrs[h], RegionData = regionData[i].Item1, Region = (int)regionData[i].Item2[r]
});
}
}
}
//parallel precalculate the crcs for the matching
Parallel.ForEach(_regionData, rgn =>
{
MemorySection rg = new MemorySection(new byte[0x20]);
rg.WriteUInt32B(0, (uint)rgn.Region);
rg.Write(0x10, rgn.RegionData);
rgn.Crc = ~Crc.Combine(Crc.Combine(~rgn.Header.CrcPreRegion, ~Crc.Compute(rg.Data), 0x20), ~rgn.Header.CrcPostRegion, 0x1fe0);
});
//Trace.WriteLine(string.Format("Crc Headers: {0} - Took: {1}", _regionData.Count.ToString(), (DateTime.Now - dt).TotalMilliseconds.ToString()));
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
DatData data = ctx.Dats;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
uint imageSize = hdr.ReadUInt32B(0x210);
string junkId = hdr.ReadString(0x214, 4);
if (junkId != "\0\0\0\0")
{
inStream.ChangeJunk(junkId);
}
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
NCrc crc = new NCrc();
NStream target = new NStream(new CryptoStream(outStream, crc, CryptoStreamMode.Write));
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - hdr.Size);
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) + (hdr.ReadUInt32B(0x428) % 4 == 0 ? 0 : 4 - (hdr.ReadUInt32B(0x428) % 4)));
long srcPos = hdr.ReadUInt32B(0x424) + fst.Size;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(junkId, hdr.ReadUInt32B(0x208)); //size that we will output from this read
crc.Snapshot("hdr.bin");
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
crc.Snapshot("fst.bin");
hdrToFst = null; //let this be collected if needed
long dstPos = hdr.ReadUInt32B(0x424) + fst.Size;
long nullsPos = dstPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, inStream.Length, hdr, fst, true, -1, out error); //result.ImageInfo.IsoSize
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(inStream.Length - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += writeGap(cf, ref nullsPos, ref srcPos, dstPos, inStream, target, true);
}
else
{
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), dstPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (dstPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)dstPos);
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)dstPos);
dstPos += copyFile(f, ref nullsPos, ref srcPos, dstPos, imageSize, inStream, target);
}
if (dstPos < imageSize)
{
dstPos += writeGap(f, ref nullsPos, ref srcPos, dstPos, inStream, target, i == 0 || i == conFiles.Count - 1);
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)ff.Length);
}
}
firstFile = false;
}
}
crc.Snapshot("files");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, false, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderGc.Read - Read and convert"); //don't let the writer lock
}
}
public BruteForceCrcResult Match(CrcItem[] crcs)
{
//reduce crcs to 3 - header, update, postUpdate
_origHeaderCrc = crcs[0].Value;
_origUpdateCrc = crcs[1].Value;
long updateSize = (crcs.Length > 2 ? crcs[2].Offset - crcs[1].Offset : 0);
uint postUpdateCrc = (crcs.Length > 2 ? crcs[2].Value : 0);
long postUpdateSize = (crcs.Length > 2 ? crcs[2].Length : 0);
for (int x = 3; x < crcs.Length; x++)
{
postUpdateCrc = ~Crc.Combine(~postUpdateCrc, ~crcs[x].Value, crcs[x].Length);
postUpdateSize += crcs[x].Length;
}
//DateTime dt = DateTime.Now;
//quick test without region brute force (which can take a while)
headerCrc headerMatch = null;
regionCrc regionMatch = null;
uint matchCrc = 0;
uint matchUpdateCrc = 0;
uint u = _updateCrcs.AsParallel().FirstOrDefault(uCrc =>
{
headerCrc m = _hdrs.AsParallel().FirstOrDefault(hdr =>
{
uint crc = ~Crc.Combine(Crc.Combine(~hdr.Crc, ~uCrc, updateSize), ~postUpdateCrc, postUpdateSize);
if (_checkCrcs.ContainsKey(crc))
{
matchCrc = crc;
headerMatch = hdr;
matchUpdateCrc = uCrc;
}
return(matchCrc != 0);
});
return(matchCrc != 0);
});
//Trace.WriteLine(string.Format("Crc Update Combos: {0} - Took: {1}", _updateCrcs.Length.ToString(), (DateTime.Now - dt).TotalMilliseconds.ToString()));
if (headerMatch == null)
{
//dt = DateTime.Now;
//region brute force tests - FirstOrDefault is used and the results are discarded as nesting them causes threading issues
uint u2 = _updateCrcs.AsParallel().FirstOrDefault(uCrc =>
{
//combine the update crc with the post update crc
uint updDataCrc = ~Crc.Combine(~uCrc, ~postUpdateCrc, postUpdateSize);
long updDataSize = updateSize + postUpdateSize;
//compare the header/region with the update and post update combines crc
regionCrc m = _regionData.AsParallel().FirstOrDefault(rgn =>
{
uint crc = ~Crc.Combine(~rgn.Crc, ~updDataCrc, updDataSize);
if (_checkCrcs.ContainsKey(crc))
{
matchCrc = crc;
regionMatch = rgn;
matchUpdateCrc = uCrc;
}
return(matchCrc != 0);
});
return(matchCrc != 0);
});
//Trace.WriteLine(string.Format("Crc Header / Update Combos: {0} - Took: {1}", (_updateCrcs.Length * _regionData.Count).ToString(), (DateTime.Now - dt).TotalMilliseconds.ToString()));
}
return(results(headerMatch, regionMatch, matchUpdateCrc, matchCrc, matchCrc == 0 ? false : _checkCrcs[matchCrc]));
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
if (!Settings.ConfigFileFound)
{
_log?.Log("!! No config file found - This is required to restore and validate images");
}
bool truncatedRvtr = false;
bool write = !(outStream is ByteStream || outStream == Stream.Null);
//ProgressResult result = ctx.Result;
byte[] origHeader = null;
byte[] dataHeader = new byte[256];
uint headerCrc = 0;
uint dataHeaderCrc = 0;
//bool fstMissingWithH3Error = false;
NCrc crc = new NCrc();
CryptoStream target = null;
int h3Errors = 0;
List <string> requiredUpdateFiles = new List <string>();
try
{
//long progress = 0;
string lastPartitionId = "";
bool generateUpdateFiller = false;
bool generateOtherFiller = true;
bool forceFillerJunk = false;
NDisc disc = new NDisc(_log, inStream);
foreach (IWiiDiscSection s in disc.EnumerateSectionsFix(generateUpdateFiller, generateOtherFiller, forceFillerJunk))
{
if (s is WiiDiscHeaderSection)
{
_hdr = (WiiDiscHeaderSection)s;
origHeader = (byte[])_hdr.Data.Clone();
target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(_hdr.Data, 0, _hdr.Data.Length); //write the header
applyPartitionTableFixes(inStream, requiredUpdateFiles, ctx.Settings, ctx.Recovery);
headerCrc = Crc.Compute(_hdr.Data);
crc.Snapshot("Header");
pc.ReaderCheckPoint1PreWrite(null, 0); //size that we will output from this read
}
else if (s is WiiPartitionSection)
{
WiiPartitionSection ps = (WiiPartitionSection)s;
if (!truncatedRvtr && ps.Header.IsRvtR && inStream.RecoverySize == NStream.FullSizeWii5)
{
_log.LogDetail(string.Format("Truncated RVT-R image detected. Pad it with 00 to {0} bytes for NKit to handle it properly", NStream.FullSizeWiiRvtr.ToString()));
truncatedRvtr = true;
}
if (applyFixes(ps.Header, inStream))
{
_hdr.UpdateRepair();
headerCrc = Crc.Compute(_hdr.Data); //recalculate
}
if (ps.Header.Type == PartitionType.Data)
{
if (applyDataPartitionFixes(ps.Header))
{
headerCrc = Crc.Compute(_hdr.Data); //recalculate
}
Array.Copy(ps.Header.Data, dataHeader, dataHeader.Length);
}
target.Write(ps.Header.Data, 0, (int)ps.Header.Size);
foreach (WiiPartitionGroupSection pg in ps.Sections)
{
if (ps.Header.Type == PartitionType.Data && pg.DiscOffset == ps.Header.DiscOffset + ps.Header.Size)
{
Array.Copy(pg.Decrypted, 0x400, dataHeader, 0, dataHeader.Length); //copy out datapartition header (256 bytes)
}
pg.Unscrub(ctx.Recovery.JunkPatches);
h3Errors += pg.H3Errors;
target.Write(pg.Encrypted, 0, (int)pg.Size);
}
if (h3Errors != 0)
{
_log?.LogDetail(string.Format("{0} unrecoverable group errors, this image will now be corrupted due to failed unscrubbing attempts!", h3Errors.ToString()));
}
lastPartitionId = ps.Id;
}
else if (s is WiiFillerSection)
{
WiiFillerSection fs = (WiiFillerSection)s;
if (fs.Size != 0)
{
foreach (WiiFillerSectionItem item in ((WiiFillerSection)s).Sections)
{
target.Write(item.Data, 0, (int)item.Size);
}
}
crc.Snapshot(((WiiFillerSection)s).DiscOffset == 0x50000 ? "[Update Filler]" : lastPartitionId);
}
}
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "RestoreReaderWii.Restore - Read and repair");
}
try
{
Tuple <string, string, string, uint>[] allParts = ctx.Recovery.WiiUPartsData.Union(ctx.Recovery.WiiUOtherPartsData).ToArray();
uint[] uniqueCrcs = allParts.Select(a => a.Item4).Union(ctx.Settings.RedumpUpdateCrcs.Where(a => !allParts.Any(b => a == b.Item4))).ToArray();
//create a data header based on the modified header
byte[] dataHdr = dataHeader;
dataHeader = (byte[])_hdr.Data.Clone();
Array.Copy(dataHdr, dataHeader, dataHdr.Length);
Array.Clear(dataHeader, 0x60, 2);
dataHeaderCrc = Crc.Compute(dataHeader);
bool isCustom = false;
bool updatePartitionMissing = false;
SortedList <uint, bool> checkCrcs = new SortedList <uint, bool>();
foreach (RedumpEntry r in ctx.Dats.RedumpData)
{
checkCrcs.Add(r.Crc, true);
}
foreach (RedumpEntry r in ctx.Dats.CustomData.Where(a => !checkCrcs.ContainsKey(a.Crc)))
{
checkCrcs.Add(r.Crc, false);
}
HeaderBruteForcer crcMtch = new HeaderBruteForcer(uniqueCrcs, checkCrcs, ctx.Settings.RedumpRegionData, _hdr.Data, dataHeader);
BruteForceCrcResult bfMatch = crcMtch.Match(crc.Crcs);
string updateFilename = allParts?.FirstOrDefault(a => a.Item4 == (bfMatch.UpdateChanged ? bfMatch.UpdateCrc : crc.Crcs[1].Value))?.Item1;
updatePartitionMissing = bfMatch.UpdateChanged && !allParts.Any(a => a.Item4 == bfMatch.UpdateCrc); //matched, but update crc not an update partition
if (bfMatch.HeaderChanged)
{
crc.Crcs[0].PatchCrc = bfMatch.HeaderCrc;
crc.Crcs[0].PatchData = bfMatch.Header;
if (bfMatch.RegionChanged)
{
_log.LogDetail(bfMatch.Region != bfMatch.OriginalRegion ? string.Format("Region changed from {0} to {1}", ((Region)bfMatch.OriginalRegion).ToString(), ((Region)bfMatch.Region).ToString()) : string.Format("Region age ratings changed for {0} region", ((Region)bfMatch.Region).ToString()));
}
}
bool isRecoverable = false;
if (bfMatch.UpdateChanged)
{
if (!updatePartitionMissing)
{
_log?.LogDetail(string.Format("Matched recovery update partition: {0}", updateFilename ?? ""));
crc.Crcs[1].Name = updateFilename ?? string.Format("[UNKNOWN {0}]", bfMatch.UpdateCrc.ToString("X8"));
crc.Crcs[1].PatchCrc = bfMatch.UpdateCrc;
crc.Crcs[1].PatchFile = Path.Combine(ctx.Recovery.WiiUPartsData.Any(a => a.Item4 == bfMatch.UpdateCrc) ? ctx.Settings.RecoveryFilesPath : ctx.Settings.OtherRecoveryFilesPath, updateFilename);
}
else
{
_log?.LogDetail(string.Format("Missing update recovery partition file *_{0}", bfMatch.UpdateCrc.ToString("X8")));
crc.Crcs[1].Name = "Missing Recovery Partition File";
crc.Crcs[1].PatchCrc = bfMatch.UpdateCrc;
crc.Crcs[1].PatchFile = null;
isRecoverable = true;
}
}
else if (!string.IsNullOrEmpty(updateFilename))
{
crc.Crcs[1].Name += string.Format(" [Matches {0}{1}]", updateFilename, isRecoverable ? " (Recoverable)" : "");
}
string resultMsg = "MatchFail";
if (bfMatch.MatchedCrc != 0)
{
if (updatePartitionMissing)
{
resultMsg = string.Format("Match {0} (Recoverable: missing update partition {1})", isCustom ? "Custom" : "Redump", bfMatch.UpdateCrc.ToString("X8"));
}
else
{
resultMsg = string.Format("Match {0}", isCustom ? "Custom" : "Redump");
}
}
pc.ReaderCheckPoint2Complete(crc, isRecoverable, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, bfMatch?.Header ?? _hdr.Data, resultMsg);
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "RestoreReaderWii.Read - Read and repair"); //don't let the writer lock
}
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
long imageSize = hdr.ReadUInt32B(0x210) * 4L;
string junkId = hdr.ReadString(0x214, 4);
uint updatePartitionCrc = hdr.ReadUInt32B(0x218);
MemorySection updateRemovedFiller = null;
long discOffset = 0;
List <NkitPartitionPatchInfo> patchInfos = new List <NkitPartitionPatchInfo>();
discOffset += hdr.Size;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
NCrc crc = new NCrc();
long dstPos = 0;
long srcPos = hdr.Size;
ScrubManager scrubFiller = new ScrubManager(null);
bool isRecoverable = false;
using (NDisc disc = new NDisc(_log, inStream))
{
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
hdr.Write8(0x60, 0);
hdr.Write8(0x61, 0);
CryptoStream crcStream = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //wrap to calculate crc
crcStream.Write(hdr.Data, 0, hdr.Data.Length); //write the header
pc.ReaderCheckPoint1PreWrite(null, nkitCrc); //size that will be output from this read
dstPos += hdr.Size;
crc.Snapshot("Disc Header");
foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted
{
if (updatePartitionCrc != 0 && srcPos == hdr.Size) //write update partition out
{
updateRemovedFiller = MemorySection.Read(inStream, hdr.Partitions.First().DiscOffset - srcPos);
srcPos += updateRemovedFiller.Size;
WiiPartitionInfo firstPart = WiiDiscHeaderSection.CreatePartitionInfos(updateRemovedFiller, 0)?.FirstOrDefault(a => a.Type != PartitionType.Update);
string updateFileName = RecoveryData.GetUpdatePartition(ctx.Settings, updatePartitionCrc);
if (updateFileName != null)
{
using (FileStream pf = File.OpenRead(updateFileName))
{
pf.Copy(crcStream, pf.Length);
dstPos += pf.Length;
}
}
else
{
string msg = string.Format("!! Update partition *_{0} missing - Adding filler. It may be Recoverable", updatePartitionCrc.ToString("X8"));
isRecoverable = true;
_log?.LogDetail(msg);
//throw pc.SetReaderException(new HandledException("Failed to convert: " + msg));
}
ByteStream.Zeros.Copy(crcStream, firstPart.DiscOffset - dstPos); //fill full gap
dstPos += firstPart.DiscOffset - dstPos;
}
NkitPartitionPatchInfo patchInfo = new NkitPartitionPatchInfo()
{
HashGroups = new Dictionary <long, MemorySection>()
};
patchInfos.Add(patchInfo);
if (part.DiscOffset > srcPos)
{
dstPos += writeFiller(ref srcPos, dstPos, dstPos + 0x1cL, inStream, crcStream, new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize), scrubFiller);
inStream.Copy(ByteStream.Zeros, part.DiscOffset - srcPos); //padded to 0x8000
srcPos += part.DiscOffset - srcPos;
}
part.DiscOffset = dstPos; //restore the original position
patchInfo.DiscOffset = dstPos;
patchInfo.PartitionHeader = MemorySection.Read(inStream, 0x20000);
srcPos += patchInfo.PartitionHeader.Size;
long size = patchInfo.PartitionHeader.ReadUInt32B(0x2bc) * 4L;
LongRef origSize = new LongRef()
{
Value = 0
};
WiiPartitionGroupSection wp = null;
WiiPartitionHeaderSection wh = new WiiPartitionHeaderSection(hdr, null, part.DiscOffset, patchInfo.PartitionHeader.Data, patchInfo.PartitionHeader.Data.Length);
MemorySection ph = new MemorySection(new byte[0x8000 * 64]);
long remaining = long.MaxValue;//set after first block read
int groupIndex = 0;
WiiHashStore hashes = new WiiHashStore();
patchInfo.ScrubManager = wh.ScrubManager;
bool patchBlock = false;
StreamCircularBuffer prtStream = null;
try
{
using (prtStream = new StreamCircularBuffer(0, null, null, output => srcPos += partitionStreamWrite(origSize, inStream, output, size, ctx.Dats, patchInfo, hashes, pc)))
{
int gs = 0;
int ge = 0;
int i = 0;
MemoryStream patchBlocks = null;
while (remaining > 0)
{
int blocks = (int)Math.Min(64L, remaining / 0x7c00);
for (int b = 0; b < blocks; b++)
{
prtStream.Read(ph.Data, (b * 0x8000) + 0x400, 0x7c00); //load aligned with no hashes
if (remaining == long.MaxValue) //first loop
{
remaining = origSize.Value;
if (ph.ReadString(0x400 + 0, 4) == "\0\0\0\0")
{
gs = -1;
ge = -1;
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, (uint)(NStream.DataToHashedLen(origSize.Value) / 4)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
}
else
{
gs = (int)((ph.ReadUInt32B(0x400 + 0x424) * 4L) / (0x7c00L * 64));
ge = (int)(((ph.ReadUInt32B(0x400 + 0x424) * 4L) + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) / (0x7c00L * 64));
if ((int)((part.DiscOffset + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) % (0x7c00L * 64)) == 0)
{
ge--; //don't load the next group if the data will end on the last byte
}
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, ph.ReadUInt32B(0x400 + 0x210)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
ph.WriteUInt32B(0x400 + 0x200, 0);
ph.WriteUInt32B(0x400 + 0x204, 0);
ph.WriteUInt32B(0x400 + 0x208, 0);
ph.WriteUInt32B(0x400 + 0x20C, 0);
ph.WriteUInt32B(0x400 + 0x210, 0);
ph.WriteUInt32B(0x400 + 0x214, 0);
ph.WriteUInt32B(0x400 + 0x218, 0);
}
wp = new WiiPartitionGroupSection(hdr, wh, ph.Data, part.DiscOffset, blocks * 0x8000, false);
wh.Initialise(wp, origSize.Value);
}
}
if (blocks < 64)
{
Array.Clear(ph.Data, blocks * 0x8000, ph.Data.Length - (blocks * 0x8000)); //clear remaining blocks
}
wp.Populate(groupIndex, ph.Data, dstPos, blocks * 0x8000);
int scrubbed = 0;
for (int bi = 0; bi < blocks; bi++)
{
wp.MarkBlockDirty(bi);
byte byt;
if (patchInfo.ScrubManager.IsBlockScrubbedScanMode(wp.Offset + (bi * 0x8000), out byt))
{
wp.SetScrubbed(bi, byt);
scrubbed++;
}
}
bool isFstBlocks = i >= gs && i <= ge;
bool reqHashes = hashes.IsPreserved(wp.Offset); //test with 0 partition based offset
repairBlocks(wp, scrubbed, blocks, false, isFstBlocks); //only test if the hashes aren't preserved (only preserved for scrubbed/customs)
if (reqHashes) //store with disc based offset
{
patchInfo.HashGroups.Add(wp.Offset + part.DiscOffset + patchInfo.PartitionHeader.Size, new MemorySection((byte[])wp.Decrypted.Clone())); //fetch the stored hashed that couldn't be recreated
}
groupIndex++;
bool inFstArea = i >= gs && i <= ge;
if (!patchBlock && (gs == i || reqHashes))
{
patchBlocks = new MemoryStream();
crc.Snapshot(lastPartitionId + " Data");
patchBlock = true;
}
else if (patchBlock && !inFstArea && !reqHashes)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
patchBlock = false;
}
#if DECRYPT
outStream.Write(wp.Decrypted, 0, blocks * 0x8000);
if (i >= gs && i <= ge)
{
fstBlocks.Write(wp.Decrypted, 0, blocks * 0x8000);
}
#else
crcStream.Write(wp.Encrypted, 0, blocks * 0x8000);
if (patchBlock)
{
patchBlocks.Write(wp.Encrypted, 0, blocks * 0x8000);
}
#endif
remaining -= (blocks * 0x7c00);
dstPos += (blocks * 0x8000);
i++;
}
if (patchBlock)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
}
}
if (origSize.Value != prtStream.WriterPosition)
{
throw pc.SetReaderException(new HandledException("Partition read did not write the full amount to the circular buffer before completing"));
}
}
catch (Exception ex)
{
if (prtStream?.WriterException != null)
{
throw pc.SetReaderException(prtStream.WriterException, "Failed reading filesystem");
}
throw pc.SetReaderException(ex, "Failed converting the filesystem");; //writer exception
}
srcPos += hashes.ReadPatchData(part.DiscOffset + patchInfo.PartitionHeader.Size, patchInfo.HashGroups, inStream);
//read hash patches
lastPartitionType = part.Type;
}
if (srcPos < inStream.Length)
{
JunkStream partJunk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
dstPos += writeFiller(ref srcPos, dstPos, lastPartitionType == PartitionType.Update ? imageSize : dstPos + 0x1cL, inStream, crcStream, partJunk, scrubFiller);
}
}
crc.Snapshot("End");
if (updatePartitionCrc != 0)
{
hdr.Write((int)WiiDiscHeaderSection.PartitionTableOffset, updateRemovedFiller.Data, 0, (int)WiiDiscHeaderSection.PartitionTableLength); //restore the exact partition table if update was removed
}
else
{
hdr.UpdateOffsets(); //just update the table with the new offsets
}
crc.Crcs[0].PatchData = hdr.Data;
foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
{
NkitPartitionPatchInfo patchInfo = patchInfos.FirstOrDefault(a => ci.Offset >= a.DiscOffset + a.PartitionHeader.Size && ci.Offset < a.DiscOffset + a.PartitionHeader.Size + a.Size);
if (patchInfo != null)
{
patchGroups(patchInfo, hdr, ci.Offset, ci.PatchData);
}
ci.PatchCrc = Crc.Compute(ci.PatchData);
}
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, isRecoverable, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - Read and convert");
}
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
string resultMsg = "";
if (!Settings.ConfigFileFound)
{
_log?.Log("!! No config file found - This is required to restore and validate images");
}
NCrc crc = new NCrc();
Settings settings = ctx.Settings;
List <string> addedFiles = new List <string>();
DatData data = ctx.Dats;
RecoveryData rec = ctx.Recovery;
List <JunkRedumpPatch> patches = rec.JunkPatches;
MemorySection hdr = null;
try
{
long junkStart = settings.JunkStartOffsets.FirstOrDefault(a => a.Id8 == inStream.Id8)?.Offset ?? 0;
string forceJunkId = settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == inStream.Id8)?.JunkId;
if (forceJunkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", forceJunkId, inStream.Id));
//result.ImageInfo.JunkId = forceJunkId;
inStream.ChangeJunk(forceJunkId);
}
if (junkStart != 0)
{
_log?.LogDetail(string.Format("Junk forced to start at 0x{0}", junkStart.ToString("X8")));
}
hdr = inStream.DiscHeader;
FstFileItem goodFst = (FstFileItem)rec.GcBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8 && !(inStream.Id8 == "GNHE5d0000" && a.Length == 116));
if (goodFst != null)
{
_log?.LogDetail(string.Format("Recovery: {0}", goodFst.Filename));
}
if (goodFst == null && rec.GcNewBinFiles != null)
{
goodFst = (FstFileItem)rec.GcNewBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8);
}
if (goodFst != null)
{
goodFst.Populate();
}
ApploaderFileItem goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
if (goodAldr == null && rec.GcNewBinFiles != null)
{
goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcNewBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
}
//is it an action replay (custom hacks)
if ((hdr.ReadUInt32B(0x420) == 0 && inStream.Id8 == "GNHE5d0000") || inStream.Id8 == "DTLX010000" || inStream.Id8 == "102E010007")
{
if (inStream.Id8 == "102E010007")
{
resultMsg = "Aging Disc detected - Skipping recover";
}
else
{
resultMsg = "Datel Action Replay detected - Skipping recover";
}
try
{
using (CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write))
{
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
inStream.Copy(target, pc.OutputSize - hdr.Size);
}
_log?.LogDetail(resultMsg);
}
catch (Exception ex)
{
throw new HandledException(ex, resultMsg);
}
crc.Snapshot("files");
}
else
{
//############################################################################
//# READ DISC START
//########### Header (boot.bin) 0 (read by base stream already)
long srcPos = hdr.Size;
_log?.LogDetail(string.Format("Header Read: {0} - {1}", friendly(inStream.Id), friendly(inStream.Title)));
//########### Header Info (bi2.bin) 0x440
MemorySection bi2bin = MemorySection.Read(inStream, 0x2000);
srcPos += bi2bin.Size;
//########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol
MemorySection appldr = MemorySection.Read(inStream, Math.Min(hdr.ReadUInt32B(0x420) == 0 ? uint.MaxValue : hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - srcPos);
srcPos += appldr.Size;
//########### APP (main.dol)
MemorySection maindol;
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424))
{
maindol = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - srcPos);
}
else
{
maindol = new MemorySection(new byte[0]);
}
srcPos += maindol.Size;
//########### FST (fst.bin)
MemorySection srcFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428));
srcPos += hdr.ReadUInt32B(0x428);
//############################################################################
//# CORRECT ISSUES
List <FstFile> srcFiles = FileSystem.Parse(srcFst, null, inStream.Id, true)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
if (srcFiles == null)
{
throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", hdr.ReadUInt32B(0x424).ToString("X8")));
}
uint crcTmp = Crc.Compute(appldr.Data, 0, Math.Min((int)appldr.Size, 0x20 + (int)(appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18))));
//adjust appldr
if (goodAldr != null && crcTmp != goodAldr.Crc)
{
_log?.LogDetail(string.Format("Replacing appldr.bin crc {0} with Recovery appldr.bin {1}", crcTmp.ToString("X8"), goodAldr.Crc.ToString("X8")));
addedFiles.Add(goodAldr.Filename);
appldr = new MemorySection(File.ReadAllBytes(goodAldr.Filename));
}
//adjust main.dol
if (goodFst != null)
{
if (goodFst.MainDolOffset != hdr.ReadUInt32B(0x420))
{
//main.dol is before fst in image and after in goodfst
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424) && goodFst.MainDolOffset > goodFst.FstOffset)
{
_log?.LogDetail(string.Format("Skipping main.dol at 0x{0} using 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
maindol = new MemorySection(new byte[0]);
}
else
{
_log?.LogDetail(string.Format("Moving main.dol address 0x{0} to 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
}
hdr.WriteUInt32B(0x420, (uint)goodFst.MainDolOffset);
}
if (goodFst.Region != (Region)bi2bin.ReadUInt32B(0x18))
{
_log?.LogDetail(string.Format("Region Changed to {0} from {1}", goodFst.Region.ToString(), ((Region)bi2bin.ReadUInt32B(0x18)).ToString()));
bi2bin.WriteUInt32B(0x18, (uint)goodFst.Region);
}
if (goodFst.MaxFst != hdr.ReadUInt32B(0x42C))
{
_log?.LogDetail(string.Format("Max Fst Size changed to {0} from {1}", goodFst.MaxFst.ToString("X8"), hdr.ReadUInt32B(0x42C).ToString("X8")));
hdr.WriteUInt32B(0x42C, (uint)goodFst.MaxFst);
}
string newTitle = Encoding.ASCII.GetString(goodFst.Title);
if (newTitle != hdr.ReadString(0x20, 0x60 - 0x20))
{
_log?.LogDetail(string.Format("Title changed to '{0}' from '{1}'", newTitle.TrimEnd('\0'), hdr.ReadString(0x20, 0x60 - 0x20).TrimEnd('\0')));
hdr.Write(0x20, goodFst.Title);
}
}
MemorySection fst = srcFst;
List <FstFile> fstFiles = srcFiles;
crcTmp = Crc.Compute(srcFst.Data);
if (goodFst != null && crcTmp != goodFst.Crc)
{
_log?.LogDetail(string.Format("Replacing fst.bin crc {0} with Recovery fst {1}", crcTmp.ToString("X8"), goodFst.Crc.ToString("X8")));
addedFiles.Add(goodFst.Filename);
fst = new MemorySection(goodFst.FstData);
fstFiles = FileSystem.Parse(fst, null, inStream.Id, true).Files.OrderBy(a => a.Offset).ThenBy(a => a.Length).ToList();
}
//adjust fst.bin
if (goodFst != null && (goodFst.FstOffset != hdr.ReadUInt32B(0x424) || goodFst.FstData.Length != hdr.ReadUInt32B(0x428)))
{
_log?.LogDetail(string.Format("Moving / Resizing fst.bin address 0x{0} (Length {1}) to 0x{2} (Length {3})", hdr.ReadUInt32B(0x424).ToString("X8"), hdr.ReadUInt32B(0x428).ToString(), goodFst.FstOffset.ToString("X8"), goodFst.FstData.Length.ToString()));
hdr.WriteUInt32B(0x424, (uint)goodFst.FstOffset);
hdr.WriteUInt32B(0x428, (uint)goodFst.FstData.Length);
}
int c;
int failCount = 0;
int filesMoved = 0;
int nkitJunkFiles = 0;
foreach (FstFile f in srcFiles)
{
FstFile[] fnd = fstFiles.Where(a => a.Name == f.Name && a.Path == f.Path).ToArray();
if (fnd.Length == 0)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: No File Found - {0}/{1} (Length {2})", f.Path, f.Name, f.Length.ToString()));
}
else if (inStream.IsNkit && (c = fnd.Count(a => a.Length != 0 && f.Length == 0)) == 1)
{
nkitJunkFiles++;
}
else if ((c = fnd.Count(a => a.Length == f.Length)) != 1)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: File Size bad - {0}/{1} (Length {2}){3}", f.Path, f.Name, f.Length.ToString(), c <= 1 ? "" : string.Format(" {0} files found", c.ToString())));
}
else if (f.DataOffset != fnd[0].DataOffset)
{
filesMoved++;
}
if (failCount >= 10)
{
break;
}
}
if (failCount != 0)
{
if (failCount >= 10)
{
throw new HandledException(string.Format("{0} or more FST errors found", failCount.ToString()));
}
}
if (filesMoved != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be repositioned when rebuilding this image", filesMoved.ToString(), filesMoved == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (nkitJunkFiles != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be generated from junk when rebuilding this image", nkitJunkFiles.ToString(), nkitJunkFiles == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (goodFst != null)
{
//is the data output so far correct
if (!bruteForceValidHeader(ctx, goodFst, hdr, bi2bin, appldr, maindol, fst, pc))
{
throw new HandledException(string.Format("Post FST Crc Failed 0x{0}", goodFst.PostFstCrc.ToString("X8")));
}
}
//############################################################################
//# WRITE DISC START
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
long dstPos = hdr.Data.Length;
crc.Snapshot("boot.bin");
target.Write(bi2bin.Data, 0, (int)bi2bin.Size);
dstPos += bi2bin.Size;
crc.Snapshot("bi2.bin");
target.Write(appldr.Data, 0, (int)appldr.Size);
dstPos += appldr.Size;
ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos);
dstPos += Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos;
crc.Snapshot("appldr.bin");
target.Write(maindol.Data, 0, (int)maindol.Size);
dstPos += maindol.Size;
if (goodFst != null)
{
long padding = goodFst.FstOffset - dstPos;
ByteStream.Zeros.Copy(target, padding);
dstPos += padding;
}
crc.Snapshot("main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
dstPos += fst.Size;
bool firstFile = true;
Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>();
crc.Snapshot("fst.bin");
//############################################################################
//# WRITE THE FILESYSTEM
int fidx = -1;
FstFile lastFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = fst.Size
};
FstFile nextFile = fstFiles[++fidx];
FstFile cacheFile = null;
long nullsPos = dstPos + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
//########### FILES
foreach (FstFile f in srcFiles) //read the files and write them out as goodFiles (possible order difference)
{
while ((cacheFile = cache.Keys.FirstOrDefault(a => (a.Length == nextFile.Length || (inStream.IsNkit && a.Length == 0)) && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) //write cache
{
bool isJunk = inStream.IsNkit && cacheFile.Length == 0 && cacheFile.Name == nextFile.Name && cacheFile.Path == nextFile.Path;
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, isJunk);
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
if (f.DataOffset - srcPos > 0) //skip src junk
{
inStream.Copy(ByteStream.Zeros, f.DataOffset - srcPos);
srcPos += f.DataOffset - srcPos;
}
if (nextFile != null && (f.Length != nextFile.Length || f.Name != nextFile.Name || f.Path != nextFile.Path)) //cache file (nkit junk files are cached)
{
bool isNkitJunk = inStream.IsNkit && f.Length == 0 && nextFile.Length != 0 && f.Name == nextFile.Name && f.Path == nextFile.Path;
byte[] cacheItem = new byte[isNkitJunk ? 8 : f.Length];
inStream.Read(cacheItem, 0, cacheItem.Length); //read the nkit junk data - details (real length and null counts)
cache.Add(f, cacheItem);
srcPos += cacheItem.Length;
}
else //copy file
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, inStream, lastFile, nextFile, patches, inStream, junkStart, false);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
srcPos += f.Length;
}
}
while (nextFile != null && fidx < fstFiles.Count && (cacheFile = cache.Keys.FirstOrDefault(a => a.Length == nextFile.Length && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null)
{
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, false);
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
writeDestGap(nullsPos, target, dstPos, pc.OutputSize - dstPos, true, patches, junkStart, inStream);
crc.Snapshot("files");
}
resultMsg = "MatchFail";
uint finalCrc = crc.FullCrc(true);
if (ctx.Dats.RedumpData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Redump";
}
if (ctx.Dats.CustomData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Custom";
}
pc.ReaderCheckPoint2Complete(crc, false, finalCrc, finalCrc, this.VerifyIsWrite, hdr.Data, resultMsg);
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "RestoreReaderGc.Read - Read and repair"); //don't let the writer lock
}
}
public override string ToString()
{
return(string.Format("{0} [{1}]", Name, Crc.ToString("X8")));
}
