private ExtractResult ExtractFilesWiiNkit(Func <ExtractedFile, bool> filter, Action <Stream, ExtractedFile> extract)
{
WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)Header;
ExtractedFile exFile = new ExtractedFile(IsGameCube ? DiscType.GameCube : DiscType.Wii,
NStream.Id8, null, 0, hdr.Size, "", "hdr.bin", ExtractedFileType.WiiDiscItem);
long srcPos = hdr.Size;
if (filter(exFile))
{
using (MemoryStream ms = new MemoryStream(hdr.Data))
{
extract(ms, exFile);
}
}
foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted
{
if (NStream.Position < part.DiscOffset)
{
NStream.Copy(Stream.Null, part.DiscOffset - NStream.Position);
}
long prtPos = NStream.Position;
MemorySection prtHdr = MemorySection.Read(NStream, 0x20000);
MemorySection prtDataHdr = MemorySection.Read(NStream, 0x440);
string prtId = prtDataHdr.ReadString(0, 4);
if (prtId != "\0\0\0\0")
{
srcPos += prtHdr.Size + prtDataHdr.Size;
exFile = new ExtractedFile(IsGameCube ? DiscType.GameCube : DiscType.Wii,
NStream.Id8, null, prtPos, prtHdr.Size, "", prtId + "hdr.bin", ExtractedFileType.WiiDiscItem);
if (filter(exFile))
{
using (MemoryStream ms = new MemoryStream(prtHdr.Data))
{
extract(ms, exFile);
}
}
extractFiles(prtId, prtDataHdr, NStream, filter, extract);
}
}
return(createExtractResult((Region)Header.ReadUInt32B(0x4e000), null));
}
internal void Initialise(WiiPartitionGroupSection firstSection)
{
_firstSection = firstSection;
IsEncrypted = firstSection.IsEncrypted;
MemorySection ms = new MemorySection(firstSection.Decrypted);
this.Id = ms.ReadString(0x400, 4);
this.DiscNo = (int)ms.Read8(0x406);
if (this.Stream != null)
{
this.Stream.ChangeJunk(this.DiscOffset + _dataOffset, this.Id, this.DiscNo, PartitionDataSize);
}
if (this.Id != "\0\0\0\0")
{
_dolOffset = ms.ReadUInt32B(0x820) * 4L; //+400 to skip hashes
FstOffset = ms.ReadUInt32B(0x824) * 4L;
FstSize = ms.ReadUInt32B(0x828) * 4L;
}
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
string resultMsg = "";
if (!Settings.ConfigFileFound)
{
_log?.Log("!! No config file found - This is required to restore and validate images");
}
NCrc crc = new NCrc();
Settings settings = ctx.Settings;
List <string> addedFiles = new List <string>();
DatData data = ctx.Dats;
RecoveryData rec = ctx.Recovery;
List <JunkRedumpPatch> patches = rec.JunkPatches;
MemorySection hdr = null;
try
{
long junkStart = settings.JunkStartOffsets.FirstOrDefault(a => a.Id8 == inStream.Id8)?.Offset ?? 0;
string forceJunkId = settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == inStream.Id8)?.JunkId;
if (forceJunkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", forceJunkId, inStream.Id));
//result.ImageInfo.JunkId = forceJunkId;
inStream.ChangeJunk(forceJunkId);
}
if (junkStart != 0)
{
_log?.LogDetail(string.Format("Junk forced to start at 0x{0}", junkStart.ToString("X8")));
}
hdr = inStream.DiscHeader;
FstFileItem goodFst = rec.GcBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8 && !(inStream.Id8 == "GNHE5d0000" && a.Length == 116));
if (goodFst != null)
{
_log?.LogDetail(string.Format("Recovery: {0}", goodFst.Filename));
}
if (goodFst == null && rec.GcNewBinFiles != null)
{
goodFst = rec.GcNewBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8);
}
if (goodFst != null)
{
goodFst.Populate();
}
ApploaderFileItem goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
if (goodAldr == null && rec.GcNewBinFiles != null)
{
goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcNewBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
}
//is it an action replay (custom hacks)
if ((hdr.ReadUInt32B(0x420) == 0 && inStream.Id8 == "GNHE5d0000") || inStream.Id8 == "DTLX010000" || inStream.Id8 == "102E010007")
{
if (inStream.Id8 == "102E010007")
{
resultMsg = "Aging Disc detected - Skipping recover";
}
else
{
resultMsg = "Datel Action Replay detected - Skipping recover";
}
try
{
using (CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write))
{
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
inStream.Copy(target, pc.OutputSize - hdr.Size);
}
_log?.LogDetail(resultMsg);
}
catch (Exception ex)
{
throw new HandledException(ex, resultMsg);
}
crc.Snapshot("files");
}
else
{
//############################################################################
//# READ DISC START
//########### Header (boot.bin) 0 (read by base stream already)
long srcPos = hdr.Size;
_log?.LogDetail(string.Format("Header Read: {0} - {1}", friendly(inStream.Id), friendly(inStream.Title)));
//########### Header Info (bi2.bin) 0x440
MemorySection bi2bin = MemorySection.Read(inStream, 0x2000);
srcPos += bi2bin.Size;
//########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol
MemorySection appldr = MemorySection.Read(inStream, Math.Min(hdr.ReadUInt32B(0x420) == 0 ? uint.MaxValue : hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - srcPos);
srcPos += appldr.Size;
//########### APP (main.dol)
MemorySection maindol;
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424))
{
maindol = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - srcPos);
}
else
{
maindol = new MemorySection(new byte[0]);
}
srcPos += maindol.Size;
//########### FST (fst.bin)
MemorySection srcFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428));
srcPos += hdr.ReadUInt32B(0x428);
//############################################################################
//# CORRECT ISSUES
List <FstFile> srcFiles = FileSystem.Parse(srcFst, null, inStream.Id, true)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
if (srcFiles == null)
{
throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", hdr.ReadUInt32B(0x424).ToString("X8")));
}
uint crcTmp = Crc.Compute(appldr.Data, 0, Math.Min((int)appldr.Size, 0x20 + (int)(appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18))));
//adjust appldr
if (goodAldr != null && crcTmp != goodAldr.Crc)
{
_log?.LogDetail(string.Format("Replacing appldr.bin crc {0} with Recovery appldr.bin {1}", crcTmp.ToString("X8"), goodAldr.Crc.ToString("X8")));
addedFiles.Add(goodAldr.Filename);
appldr = new MemorySection(File.ReadAllBytes(goodAldr.Filename));
}
//adjust main.dol
if (goodFst != null)
{
if (goodFst.MainDolOffset != hdr.ReadUInt32B(0x420))
{
//main.dol is before fst in image and after in goodfst
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424) && goodFst.MainDolOffset > goodFst.FstOffset)
{
_log?.LogDetail(string.Format("Skipping main.dol at 0x{0} using 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
maindol = new MemorySection(new byte[0]);
}
else
{
_log?.LogDetail(string.Format("Moving main.dol address 0x{0} to 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
}
hdr.WriteUInt32B(0x420, (uint)goodFst.MainDolOffset);
}
if (goodFst.Region != (Region)bi2bin.ReadUInt32B(0x18))
{
_log?.LogDetail(string.Format("Region Changed to {0} from {1}", goodFst.Region.ToString(), ((Region)bi2bin.ReadUInt32B(0x18)).ToString()));
bi2bin.WriteUInt32B(0x18, (uint)goodFst.Region);
}
if (goodFst.MaxFst != hdr.ReadUInt32B(0x42C))
{
_log?.LogDetail(string.Format("Max Fst Size changed to {0} from {1}", goodFst.MaxFst.ToString("X8"), hdr.ReadUInt32B(0x42C).ToString("X8")));
hdr.WriteUInt32B(0x42C, (uint)goodFst.MaxFst);
}
string newTitle = Encoding.ASCII.GetString(goodFst.Title);
if (newTitle != hdr.ReadString(0x20, 0x60 - 0x20))
{
_log?.LogDetail(string.Format("Title changed to '{0}' from '{1}'", newTitle.TrimEnd('\0'), hdr.ReadString(0x20, 0x60 - 0x20).TrimEnd('\0')));
hdr.Write(0x20, goodFst.Title);
}
}
MemorySection fst = srcFst;
List <FstFile> fstFiles = srcFiles;
crcTmp = Crc.Compute(srcFst.Data);
if (goodFst != null && crcTmp != goodFst.Crc)
{
_log?.LogDetail(string.Format("Replacing fst.bin crc {0} with Recovery fst {1}", crcTmp.ToString("X8"), goodFst.Crc.ToString("X8")));
addedFiles.Add(goodFst.Filename);
fst = new MemorySection(goodFst.FstData);
fstFiles = FileSystem.Parse(fst, null, inStream.Id, true).Files.OrderBy(a => a.Offset).ThenBy(a => a.Length).ToList();
}
//adjust fst.bin
if (goodFst != null && (goodFst.FstOffset != hdr.ReadUInt32B(0x424) || goodFst.FstData.Length != hdr.ReadUInt32B(0x428)))
{
_log?.LogDetail(string.Format("Moving / Resizing fst.bin address 0x{0} (Length {1}) to 0x{2} (Length {3})", hdr.ReadUInt32B(0x424).ToString("X8"), hdr.ReadUInt32B(0x428).ToString(), goodFst.FstOffset.ToString("X8"), goodFst.FstData.Length.ToString()));
hdr.WriteUInt32B(0x424, (uint)goodFst.FstOffset);
hdr.WriteUInt32B(0x428, (uint)goodFst.FstData.Length);
}
int c;
int failCount = 0;
int filesMoved = 0;
int nkitJunkFiles = 0;
foreach (FstFile f in srcFiles)
{
FstFile[] fnd = fstFiles.Where(a => a.Name == f.Name && a.Path == f.Path).ToArray();
if (fnd.Length == 0)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: No File Found - {0}/{1} (Length {2})", f.Path, f.Name, f.Length.ToString()));
}
else if (inStream.IsNkit && (c = fnd.Count(a => a.Length != 0 && f.Length == 0)) == 1)
{
nkitJunkFiles++;
}
else if ((c = fnd.Count(a => a.Length == f.Length)) != 1)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: File Size bad - {0}/{1} (Length {2}){3}", f.Path, f.Name, f.Length.ToString(), c <= 1 ? "" : string.Format(" {0} files found", c.ToString())));
}
else if (f.DataOffset != fnd[0].DataOffset)
{
filesMoved++;
}
if (failCount >= 10)
{
break;
}
}
if (failCount != 0)
{
if (failCount >= 10)
{
throw new HandledException(string.Format("{0} or more FST errors found", failCount.ToString()));
}
}
if (filesMoved != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be repositioned when rebuilding this image", filesMoved.ToString(), filesMoved == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (nkitJunkFiles != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be generated from junk when rebuilding this image", nkitJunkFiles.ToString(), nkitJunkFiles == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (goodFst != null)
{
//is the data output so far correct
if (!bruteForceValidHeader(ctx, goodFst, hdr, bi2bin, appldr, maindol, fst, pc))
{
throw new HandledException(string.Format("Post FST Crc Failed 0x{0}", goodFst.PostFstCrc.ToString("X8")));
}
}
//############################################################################
//# WRITE DISC START
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
long dstPos = hdr.Data.Length;
crc.Snapshot("boot.bin");
target.Write(bi2bin.Data, 0, (int)bi2bin.Size);
dstPos += bi2bin.Size;
crc.Snapshot("bi2.bin");
target.Write(appldr.Data, 0, (int)appldr.Size);
dstPos += appldr.Size;
ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos);
dstPos += Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos;
crc.Snapshot("appldr.bin");
target.Write(maindol.Data, 0, (int)maindol.Size);
dstPos += maindol.Size;
if (goodFst != null)
{
long padding = goodFst.FstOffset - dstPos;
ByteStream.Zeros.Copy(target, padding);
dstPos += padding;
}
crc.Snapshot("main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
dstPos += fst.Size;
bool firstFile = true;
Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>();
crc.Snapshot("fst.bin");
//############################################################################
//# WRITE THE FILESYSTEM
int fidx = -1;
FstFile lastFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = fst.Size
};
FstFile nextFile = fstFiles[++fidx];
FstFile cacheFile = null;
long nullsPos = dstPos + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
//########### FILES
foreach (FstFile f in srcFiles) //read the files and write them out as goodFiles (possible order difference)
{
while ((cacheFile = cache.Keys.FirstOrDefault(a => (a.Length == nextFile.Length || (inStream.IsNkit && a.Length == 0)) && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) //write cache
{
bool isJunk = inStream.IsNkit && cacheFile.Length == 0 && cacheFile.Name == nextFile.Name && cacheFile.Path == nextFile.Path;
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, isJunk);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
if (f.DataOffset - srcPos > 0) //skip src junk
{
inStream.Copy(ByteStream.Zeros, f.DataOffset - srcPos);
srcPos += f.DataOffset - srcPos;
}
if (nextFile != null && (f.Length != nextFile.Length || f.Name != nextFile.Name || f.Path != nextFile.Path)) //cache file (nkit junk files are cached)
{
bool isNkitJunk = inStream.IsNkit && f.Length == 0 && nextFile.Length != 0 && f.Name == nextFile.Name && f.Path == nextFile.Path;
byte[] cacheItem = new byte[isNkitJunk ? 8 : f.Length];
inStream.Read(cacheItem, 0, cacheItem.Length); //read the nkit junk data - details (real length and null counts)
cache.Add(f, cacheItem);
srcPos += cacheItem.Length;
}
else //copy file
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, inStream, lastFile, nextFile, patches, inStream, junkStart, false);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
srcPos += f.Length;
}
}
while (nextFile != null && fidx < fstFiles.Count && (cacheFile = cache.Keys.FirstOrDefault(a => a.Length == nextFile.Length && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null)
{
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, false);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
writeDestGap(nullsPos, target, dstPos, pc.OutputSize - dstPos, true, patches, junkStart, inStream);
crc.Snapshot("files");
}
resultMsg = "MatchFail";
uint finalCrc = crc.FullCrc(true);
if (ctx.Dats.RedumpData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Redump";
}
if (ctx.Dats.CustomData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Custom";
}
pc.ReaderCheckPoint2Complete(crc, false, finalCrc, finalCrc, VerifyIsWrite, hdr.Data, resultMsg);
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "RestoreReaderGc.Read - Read and repair"); //don't let the writer lock
}
}
private long partitionWrite(Stream inStream, NCrc crc, Stream target, WiiPartitionSection pHdr, Context ctx, Coordinator pc, NkitInfo imageInfo, ScrubManager scrub, WiiHashStore hashes, long fstFileAlignment)
{
#if DEHASH
return(pHdr.NewPartitionDataLength = inStream.Copy(target, pHdr.PartitionDataLength, null));
#elif DECRYPT
inStream.Copy(target, pHdr.PartitionLength, null);
return(pHdr.NewPartitionDataLength = pHdr.PartitionDataLength);
#endif
MemorySection hdr = MemorySection.Read(inStream, 0x440);
//ProgressResult result = ctx.Result;
long mlt = 4L; //GC 1L
long logicalSize = pHdr.PartitionDataLength; //GC: ctx.ImageLength;
long physicalSize = pHdr.PartitionDataLength; //GC: result.ImageInfo.IsoSize;
string junkId = hdr.ReadString(0, 4);
//bool write = true;
List <string> addedFiles = new List <string>();
long srcPos;
long dstPos = 0;
JunkStream js = new JunkStream(junkId, hdr.Read8(6), logicalSize);
try
{
if (junkId == "\0\0\0\0")
{
srcPos = hdr.Size;
imageInfo.BytesData = srcPos;
_log?.LogDetail("Null Partition ID, preserving partition as raw");
ConvertFile cf = new ConvertFile(logicalSize - hdr.Size, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.Size, Offset = hdr.Size, Length = 0
},
};
long nullsPos = 0;
target.Write(hdr.Data, 0, (int)hdr.Size); //0x400
target.Write(pHdr.Header.Data, 0x2bc, 4); //copy the original partition length
dstPos += 0x440 + 4 + NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
hdr.WriteString(0x200, 8, "NKIT v01");
hdr.WriteUInt32B(0x210, (uint)(pHdr.PartitionLength / mlt));
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
crc.Snapshot(junkId + " PrtHdr");
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot(junkId + " Fst");
target.Write(hashes.FlagsToByteArray(), 0, (int)hashes.FlagsLength);
crc.Snapshot(junkId + " HashFlags");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos + hashes.FlagsLength;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, physicalSize, hdr, fst, false, fstFileAlignment, out error);
imageInfo.BytesData = srcPos;
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(pHdr.PartitionDataLength - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
List <ConvertFile> missing;
NkitFormat.NkitWriteFileSystem(ctx, imageInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out missing, scrub, pHdr.PartitionDataLength, _log);
dstPos += hashes.HashesToStream(target);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
crc.Crcs[crc.Crcs.Length - 1].PatchData = hashes.FlagsToByteArray();
crc.Crcs[crc.Crcs.Length - 1].PatchCrc = Crc.Compute(crc.Crcs[crc.Crcs.Length - 2].PatchData);
crc.Crcs[crc.Crcs.Length - 2].PatchData = fst.Data;
crc.Crcs[crc.Crcs.Length - 2].PatchCrc = Crc.Compute(fst.Data);
}
}
}
catch (Exception ex)
{
throw new HandledException(ex, "NkitWriterGc.Write - Convert");
}
return(dstPos);
}
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
{
try
{
long mlt = 1L; //for Wii: 4L
long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength
pc.WriterCheckPoint1WriteReady(out string junkId); //wait until read has written the header and set the length
List <string> addedFiles = new List <string>();
NCrc crc = new NCrc();
long srcPos;
long dstPos = 0;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
string id8 = string.Concat(hdr.ReadString(0, 6), hdr.Data[6].ToString("X2"), hdr.Data[7].ToString("X2"));
if (junkId == null)
{
junkId = ctx.Settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == id8)?.JunkId;
if (junkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", junkId, id8.Substring(0, 4)));
}
}
if (junkId == null)
{
junkId = hdr.ReadString(0, 4);
}
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
long fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == id8)?.Item2 ?? -1;
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
crc.Snapshot("hdr.bin");
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot("fst.bin");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
JunkStream js = new JunkStream(junkId, hdr.Read8(6), NStream.FullSizeGameCube);
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, imageSize, hdr, fst, true, fstFileAlignment, out string error); //Size isn't important for writing //result.ImageInfo.IsoSize
NkitInfo nkitInfo = new NkitInfo
{
BytesData = srcPos,
BytesGaps = 0,
BytesJunkFiles = 0,
BytesPreservationData = 0,
BytesPreservationDiscPadding = 0
};
ScrubManager scrub = new ScrubManager();
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
NkitFormat.NkitWriteFileSystem(ctx, nkitInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out List <ConvertFile> missing, scrub, imageSize, _log);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
}
if (dstPos % 0x800 != 0)
{
long l = 0x800 - (dstPos % 0x800);
ByteStream.Zeros.Copy(target, l);
dstPos += l;
nkitInfo.BytesPreservationDiscPadding += l;
}
crc.Snapshot("files");
NkitFormat.LogNkitInfo(nkitInfo, _log, hdr.ReadString(0, 4), true);
pc.WriterCheckPoint2Complete(out NCrc readerCrcs, out uint validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.
hdr.WriteUInt32B(0x420, (uint)mainDolAddr);
hdr.WriteString(0x200, 8, "NKIT v01"); //header and version
hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
hdr.WriteUInt32B(0x210, (uint)imageSize); //result.ImageInfo.IsoSize);
hdr.WriteString(0x214, 4, hdr.ReadString(0, 4) != junkId ? junkId : "\0\0\0\0");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, "NKit Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "NkitWriterGc.Write - Convert");
}
}
public virtual OutputResults Process(Context ctx, NStream input, Stream output)
{
OutputResults results = new OutputResults()
{
Conversion = ctx.ConversionName, VerifyOutputResult = VerifyResult.Unverified, ValidateReadResult = VerifyResult.Unverified
};
try
{
StreamCircularBuffer fs = null;
_timer = new Timer
{
Interval = 250
};
_timer.Elapsed += (s, e) =>
{
if (_timerRunning)
{
return; //keep processing
}
try
{
_timerRunning = true;
_log.ProcessingProgress(((IProgress)fs)?.Value ?? 0);
}
catch { }
finally
{
_timerRunning = false;
}
};
_timer.Enabled = true;
long size;
switch (_sizeMode)
{
case ProcessorSizeMode.Source:
size = input.SourceSize;
break;
case ProcessorSizeMode.Stream:
size = input.Length;
break;
case ProcessorSizeMode.Image:
size = input.ImageSize;
break;
case ProcessorSizeMode.Recover:
default:
size = input.RecoverySize;
break;
}
Coordinator pc = new Coordinator(ctx.ValidationCrc, Reader, Writer, size);
pc.Started += (s, e) =>
{
_timer.Enabled = true;
results.AliasJunkId = e.AliasJunkId;
};
pc.Completed += (s, e) =>
{
_timer.Enabled = false;
if (Writer is HashWriter)
{
results.OutputMd5 = e.Md5;
results.OutputSha1 = e.Sha1;
}
else
{
MemorySection hdr = new MemorySection(e.Header ?? input.DiscHeader.Data);
results.OutputTitle = hdr.ReadStringToNull(0x20, 0x60);
results.OutputDiscNo = hdr.Read8(6);
results.OutputDiscVersion = hdr.Read8(7);
results.OutputId8 = string.Concat(hdr.ReadString(0, 6), results.OutputDiscNo.ToString("X2"), results.OutputDiscVersion.ToString("X2"));
results.ProcessorMessage = e.ResultMessage;
results.OutputCrc = e.PatchedCrc;
results.IsRecoverable = e.IsRecoverable;
if (e.ValidationCrc != 0)
{
results.ValidationCrc = e.ValidationCrc;
results.ValidateReadResult = e.ValidationCrc == e.PatchedCrc ? VerifyResult.VerifySuccess : VerifyResult.VerifyFailed;
}
if (!(Writer is VerifyWriter))
{
results.OutputSize = e.OutputSize; //never store the verify size
}
else
{
results.VerifyCrc = e.VerifyCrc;
if (e.VerifyIsWrite) //e.ValidationCrc can be set from a previous process run
{
results.VerifyOutputResult = results.ValidationCrc == results.VerifyCrc ? VerifyResult.VerifySuccess : VerifyResult.VerifyFailed;
}
}
}
bool l9 = pc.Patches.Crcs.Any(a => a.Offset > 0xFFFFFFFFL || a.Length > 0xFFFFFFFFL);
if (pc.ReaderCrcs != null)
{
foreach (CrcItem c in pc.ReaderCrcs.Crcs)
{
_log.LogDebug(string.Format("R-CRC {0} Before:{1} After:{2} L:{3} {4}", c.Offset.ToString(l9 ? "X9" : "X8"), c.Value.ToString("X8"), c.PatchCrc == 0 ? " " : c.PatchCrc.ToString("X8"), c.Length.ToString(l9 ? "X9" : "X8"), SourceFiles.CleanseFileName(c.Name)));
}
_log.LogDebug(string.Format("ReadCRC {0}Before:{1} After:{2}", l9 ? " " : "", pc.ReaderCrcs.FullCrc(false).ToString("X8"), pc.ReaderCrcs.FullCrc(true).ToString("X8")));
}
if (pc.WriterCrcs != null)
{
foreach (CrcItem c in pc.WriterCrcs.Crcs)
{
_log.LogDebug(string.Format("W-CRC {0} Before:{1} After:{2} L:{3} {4}", c.Offset.ToString(l9 ? "X9" : "X8"), c.Value.ToString("X8"), c.PatchCrc == 0 ? " " : c.PatchCrc.ToString("X8"), c.Length.ToString(l9 ? "X9" : "X8"), SourceFiles.CleanseFileName(c.Name)));
}
_log.LogDebug(string.Format("WriteCRC {0}Before:{1} After:{2}", l9 ? " " : "", pc.WriterCrcs.FullCrc(false).ToString("X8"), pc.WriterCrcs.FullCrc(true).ToString("X8")));
}
_log.ProcessingComplete(results.OutputSize, results.ProcessorMessage, true);
};
try
{
_log.ProcessingStart(input.SourceSize, Title);
using (fs = new StreamCircularBuffer(size, input, null, s => Reader.Read(ctx, input, s, pc))) //read in stream and write to circular buffer
{
Writer.Write(ctx, fs, output, pc);
}
}
catch
{
if (pc.Exception != null)
{
throw pc.Exception;
}
if (fs.WriterException != null)
{
throw fs.WriterException;
}
throw; //writer exception
}
foreach (CrcItem crc in pc.Patches.Crcs.Where(a => a.PatchData != null || a.PatchFile != null))
{
output.Seek(crc.Offset, SeekOrigin.Begin);
if (crc.PatchFile == null)
{
output.Write(crc.PatchData, 0, (int)Math.Min(crc.Length, crc.PatchData.Length)); //PatchData might be larger
}
else
{
using (FileStream pf = File.OpenRead(crc.PatchFile))
{
pf.Copy(output, pf.Length);
ByteStream.Zeros.Copy(output, crc.Length - pf.Length);
}
}
}
}
catch (Exception ex)
{
if (_timer != null)
{
_timer.Enabled = false;
}
try
{
_log.ProcessingComplete(results.OutputSize, results.ProcessorMessage, false); // force any log lines to be output - handy for diagnosis
}
catch { }
throw new HandledException(ex, "Failed processing {0} -> {1}", Reader?.GetType()?.Name ?? "<null>", Writer?.GetType()?.Name ?? "<null>");
}
finally
{
if (_timer != null)
{
_timer.Enabled = false;
_timer = null;
}
}
return(results);
}
internal static List <ConvertFile> GetConvertFstFiles(Stream inStream, long size, MemorySection hdr, MemorySection fst, bool isGc, long fstFileAlignment, out string error)
{
string[] align = new string[] { ".tgc" }; //, ".dol", ".thp", ".adp", ".poo", ".pcm", ".pcm16", ".son", ".act", ".bin", ".gpl", ".tpl", ".bmp", ".sam", ".sdi", ".tcs", ".txt", ".skn", ".stp", ".anm" };
error = null;
List <ConvertFile> conFiles = new List <ConvertFile>();
try
{
List <FstFile> srcFiles = FileSystem.Parse(fst, null, hdr.ReadString(0, 4), isGc)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
//get list of files and gaps
long end;
long gap;
long fstLen = hdr.ReadUInt32B(0x424) * (isGc ? 1L : 4L);
FstFile ff = new FstFile(null)
{
Name = "fst.bin", DataOffset = fstLen, Offset = fstLen, Length = (int)fst.Size, IsNonFstFile = true
};
for (int i = 0; i < srcFiles.Count; i++)
{
ff = i == 0 ? ff : srcFiles[i - 1];
end = ff.DataOffset + ff.Length;
end += end % 4 == 0 ? 0 : 4 - (end % 4);
gap = srcFiles[i].DataOffset - end;
if (gap < 0)
{
error = string.Format("The gap between '{0}' and '{1}' is {2} - Converting as bad image", ff.Name, srcFiles[i].Name, gap.ToString());
return(null);
}
conFiles.Add(new ConvertFile(gap, isGc)
{
FstFile = ff
});
}
ff = srcFiles.Last();
end = ff.DataOffset + ff.Length;
end += end % 4 == 0 ? 0 : 4 - (end % 4);
gap = size - end;
if (gap >= -3 && gap < 0)
{
gap = 0; //some hacked gc images converted from tgc end on the file end (star fox e3)
}
if (gap < 0)
{
error = string.Format("The gap between '{0}' and the end of the image is {1} - Converting as bad image/partition", ff.Name, gap.ToString());
return(null);
}
conFiles.Add(new ConvertFile(gap, isGc)
{
FstFile = ff
});
//set alignment
foreach (ConvertFile cf in conFiles)
{
ff = cf.FstFile;
if (fstFileAlignment == 0)
{
cf.Alignment = 0; //preserve alignment
}
else if (fstFileAlignment == -1 && ff.DataOffset % 0x8000 == 0 && (ff.Length % 0x8000 == 0 || align.Contains(Path.GetExtension(ff.Name).ToLower())))
{
cf.Alignment = 0x8000; //default behaviour
}
else if (fstFileAlignment != 0 && ff.DataOffset % fstFileAlignment == 0) //src matches alignment
{
cf.Alignment = fstFileAlignment; //align to largest multiple
}
else
{
cf.Alignment = -1; //-1 = do not align this file
}
}
}
catch
{
error = "Fst parsing error - Converting as bad image";
return(null);
}
return(conFiles);
}
public ExtractResult ExtractRecoveryFilesGc()
{
MemorySection hdr = NStream.DiscHeader;
MemorySection bi2bin;
List <ExtractRecoveryResult> result = new List <ExtractRecoveryResult>();
NCrc crc = new NCrc();
uint appLdrCrc = 0;
MemorySection fst = null;
uint fstCrc = 0;
string fn;
string tmpFullName;
Log?.Log(string.Format("Processing: {0}", SourceFileName));
int storeType;
using (CryptoStream bw = new CryptoStream(ByteStream.Zeros, crc, CryptoStreamMode.Write))
{
bw.Write(hdr.Data, 0, (int)hdr.Size);
bi2bin = MemorySection.Read(NStream, 0x2000);
bw.Write(bi2bin.Data, 0, (int)bi2bin.Size);
MemorySection aplHdr = MemorySection.Read(NStream, 0x20);
byte[] appLdr = new byte[0x20 + aplHdr.ReadUInt32B(0x14) + aplHdr.ReadUInt32B(0x18)];
Array.Copy(aplHdr.Data, appLdr, (int)aplHdr.Size);
NStream.Read(appLdr, (int)aplHdr.Size, appLdr.Length - (int)aplHdr.Size);
appLdrCrc = Crc.Compute(appLdr);
fn = string.Format("appldr[{0}][{1}].bin", aplHdr.ReadString(0, 10).Replace("/", ""), appLdrCrc.ToString("X8"));
Log?.Log(string.Format(" 1 of 2 - Extracting appldr.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + appLdrCrc.ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, appLdr);
if ((storeType = storeRecoveryFile(this.Settings.RedumpAppldrCrcs.Contains(appLdrCrc), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
bw.Write(appLdr, 0, appLdr.Length); //add to fullcrc
NStream.Copy(bw, hdr.ReadUInt32B(0x424) - (0x2440 + appLdr.Length));
byte[] fstData = new byte[(int)((4 * 4) + (0x60 - 0x20) + hdr.ReadUInt32B(0x428))];
fst = new MemorySection(fstData); //info: dolAddr, fstAddr, maxfst, region, title, fst
NStream.Read(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
fst.WriteUInt32B(0x00, hdr.ReadUInt32B(0x420)); //dol
fst.WriteUInt32B(0x04, hdr.ReadUInt32B(0x424)); //fstAddr
fst.WriteUInt32B(0x08, hdr.ReadUInt32B(0x42C)); //maxfst
fst.WriteUInt32B(0x0c, bi2bin.ReadUInt32B(0x18)); //region
fst.Write(0x10, hdr.Read(0x20, 0x60 - 0x20)); //title
fstCrc = Crc.Compute(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428)); //fst
bw.Write(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
crc.Snapshot("crc");
}
//fst checksums are postfst
fn = string.Format("fst[{0}][{1}][{2}][{3}].bin", NStream.Id8, appLdrCrc.ToString("X8"), fstCrc.ToString("X8"), crc.FullCrc().ToString("X8"));
Log?.Log(string.Format(" 2 of 2 - Extracting fst.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + crc.FullCrc().ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, fst.Data);
if ((storeType = storeRecoveryFile(this.Settings.RedumpFstCrcs.Contains(crc.FullCrc()), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
return(createExtractResult((Region)bi2bin.Read8(0x18), result.ToArray()));
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
DatData data = ctx.Dats;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
uint imageSize = hdr.ReadUInt32B(0x210);
string junkId = hdr.ReadString(0x214, 4);
if (junkId != "\0\0\0\0")
{
inStream.ChangeJunk(junkId);
}
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
NCrc crc = new NCrc();
NStream target = new NStream(new CryptoStream(outStream, crc, CryptoStreamMode.Write));
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - hdr.Size);
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) + (hdr.ReadUInt32B(0x428) % 4 == 0 ? 0 : 4 - (hdr.ReadUInt32B(0x428) % 4)));
long srcPos = hdr.ReadUInt32B(0x424) + fst.Size;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(junkId, hdr.ReadUInt32B(0x208)); //size that we will output from this read
crc.Snapshot("hdr.bin");
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
crc.Snapshot("fst.bin");
hdrToFst = null; //let this be collected if needed
long dstPos = hdr.ReadUInt32B(0x424) + fst.Size;
long nullsPos = dstPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, inStream.Length, hdr, fst, true, -1, out error); //result.ImageInfo.IsoSize
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(inStream.Length - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += writeGap(cf, ref nullsPos, ref srcPos, dstPos, inStream, target, true);
}
else
{
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), dstPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (dstPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)dstPos);
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)dstPos);
dstPos += copyFile(f, ref nullsPos, ref srcPos, dstPos, imageSize, inStream, target);
}
if (dstPos < imageSize)
{
dstPos += writeGap(f, ref nullsPos, ref srcPos, dstPos, inStream, target, i == 0 || i == conFiles.Count - 1);
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)ff.Length);
}
}
firstFile = false;
}
}
crc.Snapshot("files");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, false, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderGc.Read - Read and convert"); //don't let the writer lock
}
}
private long partitionStreamWrite(LongRef outSize, Stream inStream, Stream target, long size, DatData settingsData, NkitPartitionPatchInfo patchInfo, WiiHashStore hashes, Coordinator pc)
{
DatData data = settingsData;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
long srcPos = hdr.Size;
long outPos = 0;
long imageSize = 0;
try
{
if (hdr.ReadString(0, 4) == "\0\0\0\0")
{
long nullsPos = 0;
long fileLength = -1;
LongRef gapLength = new LongRef()
{
Value = -1
};
target.Write(hdr.Data, 0, (int)hdr.Size);
MemorySection sz = MemorySection.Read(inStream, 4);
srcPos += 4;
outPos += hdr.Size;
imageSize = sz.ReadUInt32B(0) * 4L;
outSize.Value = NStream.HashedLenToData(imageSize);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), outSize.Value); //SET LENGTH FROM HEADER
outPos += writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
imageSize = NStream.HashedLenToData((hdr.ReadUInt32B(0x210) * 4L));
outSize.Value = imageSize;
string junkId = hdr.ReadString(0x214, 4);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), imageSize); //SET LENGTH FROM HEADER
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x424) * 4L) - hdr.Size);
srcPos += hdrToFst.Size;
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) * 4L);
long postFstPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
srcPos += fst.Size;
hashes.WriteFlagsData(imageSize, inStream);
srcPos += hashes.FlagsLength;
patchInfo.PartitionDataHeader = hdr;
patchInfo.Fst = fst;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
target.Write(fst.Data, 0, fst.Data.Length);
hdrToFst = null; //let this be collected if needed
outPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
long nullsPos = outPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, size, hdr, fst, false, -1, out error);
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
outPos += writeGap(cf, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
conFiles[0].GapLength -= hashes.FlagsLength; //fix for a few customs (no gap between the fst and the first file on the source image, but the hash mask makes it look like there is)
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), outPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (outPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset) //skip any padding (not written for wii currently)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)(outPos / 4L));
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(outPos / 4L));
outPos += copyFile(f, ref nullsPos, ref srcPos, outPos, inStream, target);
}
if (outPos < imageSize)
{
long gapLen = writeGap(f, ref nullsPos, ref srcPos, outPos, inStream, target, junk, i == 0 || i == conFiles.Count - 1, patchInfo.ScrubManager);
outPos += gapLen;
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)(ff.Length));
}
}
firstFile = false;
}
}
}
return(srcPos);
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - partitionRead");
}
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
long imageSize = hdr.ReadUInt32B(0x210) * 4L;
string junkId = hdr.ReadString(0x214, 4);
uint updatePartitionCrc = hdr.ReadUInt32B(0x218);
MemorySection updateRemovedFiller = null;
long discOffset = 0;
List <NkitPartitionPatchInfo> patchInfos = new List <NkitPartitionPatchInfo>();
discOffset += hdr.Size;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
NCrc crc = new NCrc();
long dstPos = 0;
long srcPos = hdr.Size;
ScrubManager scrubFiller = new ScrubManager(null);
bool isRecoverable = false;
using (NDisc disc = new NDisc(_log, inStream))
{
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
hdr.Write8(0x60, 0);
hdr.Write8(0x61, 0);
CryptoStream crcStream = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //wrap to calculate crc
crcStream.Write(hdr.Data, 0, hdr.Data.Length); //write the header
pc.ReaderCheckPoint1PreWrite(null, nkitCrc); //size that will be output from this read
dstPos += hdr.Size;
crc.Snapshot("Disc Header");
foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted
{
if (updatePartitionCrc != 0 && srcPos == hdr.Size) //write update partition out
{
updateRemovedFiller = MemorySection.Read(inStream, hdr.Partitions.First().DiscOffset - srcPos);
srcPos += updateRemovedFiller.Size;
WiiPartitionInfo firstPart = WiiDiscHeaderSection.CreatePartitionInfos(updateRemovedFiller, 0)?.FirstOrDefault(a => a.Type != PartitionType.Update);
string updateFileName = RecoveryData.GetUpdatePartition(ctx.Settings, updatePartitionCrc);
if (updateFileName != null)
{
using (FileStream pf = File.OpenRead(updateFileName))
{
pf.Copy(crcStream, pf.Length);
dstPos += pf.Length;
}
}
else
{
string msg = string.Format("!! Update partition *_{0} missing - Adding filler. It may be Recoverable", updatePartitionCrc.ToString("X8"));
isRecoverable = true;
_log?.LogDetail(msg);
//throw pc.SetReaderException(new HandledException("Failed to convert: " + msg));
}
ByteStream.Zeros.Copy(crcStream, firstPart.DiscOffset - dstPos); //fill full gap
dstPos += firstPart.DiscOffset - dstPos;
}
NkitPartitionPatchInfo patchInfo = new NkitPartitionPatchInfo()
{
HashGroups = new Dictionary <long, MemorySection>()
};
patchInfos.Add(patchInfo);
if (part.DiscOffset > srcPos)
{
dstPos += writeFiller(ref srcPos, dstPos, dstPos + 0x1cL, inStream, crcStream, new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize), scrubFiller);
inStream.Copy(ByteStream.Zeros, part.DiscOffset - srcPos); //padded to 0x8000
srcPos += part.DiscOffset - srcPos;
}
part.DiscOffset = dstPos; //restore the original position
patchInfo.DiscOffset = dstPos;
patchInfo.PartitionHeader = MemorySection.Read(inStream, 0x20000);
srcPos += patchInfo.PartitionHeader.Size;
long size = patchInfo.PartitionHeader.ReadUInt32B(0x2bc) * 4L;
LongRef origSize = new LongRef()
{
Value = 0
};
WiiPartitionGroupSection wp = null;
WiiPartitionHeaderSection wh = new WiiPartitionHeaderSection(hdr, null, part.DiscOffset, patchInfo.PartitionHeader.Data, patchInfo.PartitionHeader.Data.Length);
MemorySection ph = new MemorySection(new byte[0x8000 * 64]);
long remaining = long.MaxValue;//set after first block read
int groupIndex = 0;
WiiHashStore hashes = new WiiHashStore();
patchInfo.ScrubManager = wh.ScrubManager;
bool patchBlock = false;
StreamCircularBuffer prtStream = null;
try
{
using (prtStream = new StreamCircularBuffer(0, null, null, output => srcPos += partitionStreamWrite(origSize, inStream, output, size, ctx.Dats, patchInfo, hashes, pc)))
{
int gs = 0;
int ge = 0;
int i = 0;
MemoryStream patchBlocks = null;
while (remaining > 0)
{
int blocks = (int)Math.Min(64L, remaining / 0x7c00);
for (int b = 0; b < blocks; b++)
{
prtStream.Read(ph.Data, (b * 0x8000) + 0x400, 0x7c00); //load aligned with no hashes
if (remaining == long.MaxValue) //first loop
{
remaining = origSize.Value;
if (ph.ReadString(0x400 + 0, 4) == "\0\0\0\0")
{
gs = -1;
ge = -1;
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, (uint)(NStream.DataToHashedLen(origSize.Value) / 4)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
}
else
{
gs = (int)((ph.ReadUInt32B(0x400 + 0x424) * 4L) / (0x7c00L * 64));
ge = (int)(((ph.ReadUInt32B(0x400 + 0x424) * 4L) + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) / (0x7c00L * 64));
if ((int)((part.DiscOffset + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) % (0x7c00L * 64)) == 0)
{
ge--; //don't load the next group if the data will end on the last byte
}
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, ph.ReadUInt32B(0x400 + 0x210)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
ph.WriteUInt32B(0x400 + 0x200, 0);
ph.WriteUInt32B(0x400 + 0x204, 0);
ph.WriteUInt32B(0x400 + 0x208, 0);
ph.WriteUInt32B(0x400 + 0x20C, 0);
ph.WriteUInt32B(0x400 + 0x210, 0);
ph.WriteUInt32B(0x400 + 0x214, 0);
ph.WriteUInt32B(0x400 + 0x218, 0);
}
wp = new WiiPartitionGroupSection(hdr, wh, ph.Data, part.DiscOffset, blocks * 0x8000, false);
wh.Initialise(wp, origSize.Value);
}
}
if (blocks < 64)
{
Array.Clear(ph.Data, blocks * 0x8000, ph.Data.Length - (blocks * 0x8000)); //clear remaining blocks
}
wp.Populate(groupIndex, ph.Data, dstPos, blocks * 0x8000);
int scrubbed = 0;
for (int bi = 0; bi < blocks; bi++)
{
wp.MarkBlockDirty(bi);
byte byt;
if (patchInfo.ScrubManager.IsBlockScrubbedScanMode(wp.Offset + (bi * 0x8000), out byt))
{
wp.SetScrubbed(bi, byt);
scrubbed++;
}
}
bool isFstBlocks = i >= gs && i <= ge;
bool reqHashes = hashes.IsPreserved(wp.Offset); //test with 0 partition based offset
repairBlocks(wp, scrubbed, blocks, false, isFstBlocks); //only test if the hashes aren't preserved (only preserved for scrubbed/customs)
if (reqHashes) //store with disc based offset
{
patchInfo.HashGroups.Add(wp.Offset + part.DiscOffset + patchInfo.PartitionHeader.Size, new MemorySection((byte[])wp.Decrypted.Clone())); //fetch the stored hashed that couldn't be recreated
}
groupIndex++;
bool inFstArea = i >= gs && i <= ge;
if (!patchBlock && (gs == i || reqHashes))
{
patchBlocks = new MemoryStream();
crc.Snapshot(lastPartitionId + " Data");
patchBlock = true;
}
else if (patchBlock && !inFstArea && !reqHashes)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
patchBlock = false;
}
#if DECRYPT
outStream.Write(wp.Decrypted, 0, blocks * 0x8000);
if (i >= gs && i <= ge)
{
fstBlocks.Write(wp.Decrypted, 0, blocks * 0x8000);
}
#else
crcStream.Write(wp.Encrypted, 0, blocks * 0x8000);
if (patchBlock)
{
patchBlocks.Write(wp.Encrypted, 0, blocks * 0x8000);
}
#endif
remaining -= (blocks * 0x7c00);
dstPos += (blocks * 0x8000);
i++;
}
if (patchBlock)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
}
}
if (origSize.Value != prtStream.WriterPosition)
{
throw pc.SetReaderException(new HandledException("Partition read did not write the full amount to the circular buffer before completing"));
}
}
catch (Exception ex)
{
if (prtStream?.WriterException != null)
{
throw pc.SetReaderException(prtStream.WriterException, "Failed reading filesystem");
}
throw pc.SetReaderException(ex, "Failed converting the filesystem");; //writer exception
}
srcPos += hashes.ReadPatchData(part.DiscOffset + patchInfo.PartitionHeader.Size, patchInfo.HashGroups, inStream);
//read hash patches
lastPartitionType = part.Type;
}
if (srcPos < inStream.Length)
{
JunkStream partJunk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
dstPos += writeFiller(ref srcPos, dstPos, lastPartitionType == PartitionType.Update ? imageSize : dstPos + 0x1cL, inStream, crcStream, partJunk, scrubFiller);
}
}
crc.Snapshot("End");
if (updatePartitionCrc != 0)
{
hdr.Write((int)WiiDiscHeaderSection.PartitionTableOffset, updateRemovedFiller.Data, 0, (int)WiiDiscHeaderSection.PartitionTableLength); //restore the exact partition table if update was removed
}
else
{
hdr.UpdateOffsets(); //just update the table with the new offsets
}
crc.Crcs[0].PatchData = hdr.Data;
foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
{
NkitPartitionPatchInfo patchInfo = patchInfos.FirstOrDefault(a => ci.Offset >= a.DiscOffset + a.PartitionHeader.Size && ci.Offset < a.DiscOffset + a.PartitionHeader.Size + a.Size);
if (patchInfo != null)
{
patchGroups(patchInfo, hdr, ci.Offset, ci.PatchData);
}
ci.PatchCrc = Crc.Compute(ci.PatchData);
}
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, isRecoverable, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - Read and convert");
}
}