private bool bruteForceValidHeader(Context ctx, FstFileItem goodFst, MemorySection hdr, MemorySection bi2bin, MemorySection appldr, MemorySection maindol, MemorySection fst, Coordinator pc) { int hdrScrub = 0; int bi2Scrub = 0; int maindolScrub = 0; foreach (int scrub in new int[] { 0, 1, 2 }) { if (scrub == 1) { hdrScrub = scrubData(hdr.Data, 0x300, 4); bi2Scrub = scrubData(bi2bin.Data, 0x500 - 0x440, 4); if (hdrScrub == 0 && bi2Scrub == 0) { continue; //no change } } if (scrub == 2) { hdrScrub = scrubData(hdr.Data, 0x80, 0x3F0 - 0x80); bi2Scrub = scrubData(bi2bin.Data, 0x30, bi2bin.Data.Length - (0x30 + 0x60)); //0x30 at start and 0x50 from end (conflict I, II + Splinter cell have 0x40 at end) if (maindol.Size != 0) { uint maindolSize = 0; for (int i = 0; i < 18; i++) { if (maindol.ReadUInt32B(0x0 + (i * 4)) != 0) //7 text offsets, 11 data offsets { maindolSize = Math.Max(maindolSize, maindol.ReadUInt32B(0x0 + (i * 4)) + maindol.ReadUInt32B(0x90 + (i * 4))); } } maindolScrub = scrubData(maindol.Data, (int)maindolSize, (int)(maindol.Data.Length - maindolSize)); } if (hdrScrub == 0 && bi2Scrub == 0 && maindolScrub == 0) { continue; //no change } } NCrc crc = new NCrc(); MemoryStream ms = new MemoryStream(); using (CryptoStream target = new CryptoStream(ms, crc, CryptoStreamMode.Write)) { target.Write(hdr.Data, 0, (int)hdr.Size); long dstPos = hdr.Data.Length; target.Write(bi2bin.Data, 0, (int)bi2bin.Size); dstPos += bi2bin.Size; target.Write(appldr.Data, 0, (int)appldr.Size); dstPos += appldr.Size; ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), goodFst.FstOffset) - dstPos); dstPos += Math.Min(hdr.ReadUInt32B(0x420), goodFst.FstOffset) - dstPos; target.Write(maindol.Data, 0, Math.Min((int)(goodFst.FstOffset - dstPos), (int)maindol.Size)); dstPos += Math.Min((int)(goodFst.FstOffset - dstPos), (int)maindol.Size); long padding = goodFst.FstOffset - dstPos; ByteStream.Zeros.Copy(target, padding); dstPos += padding; target.Write(fst.Data, 0, fst.Data.Length); dstPos += fst.Size; Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>(); crc.Snapshot("header"); } if (crc.FullCrc() == goodFst.PostFstCrc) { if (hdrScrub != 0 || bi2Scrub != 0 || maindolScrub != 0) { //ctx.Result.GameCube.UpdatedHeader = true; _log?.LogDetail("Header brute forced to match"); if (scrub == 1) { _log?.LogDetail(string.Format(" Bytes scrubbed at 0x300 and 0x500", hdrScrub.ToString(), bi2Scrub.ToString())); } else if (scrub == 2) { _log?.LogDetail(string.Format(" Bytes scrubbed in hdr.bin ({0}), bi2.bin ({1}), main.dol ({2})", hdrScrub.ToString(), bi2Scrub.ToString(), maindolScrub.ToString())); } } return(true); } } return(false); }
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc) { NCrc crc = new NCrc(); try { CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write); uint outputCrc = 0; if (inStream.IsNkit || inStream.IsGameCube || !EncryptWiiPartitions) { pc.ReaderCheckPoint1PreWrite(null, 0); //size that we will output from this read if (inStream.HeaderRead) { target.Write(inStream.DiscHeader.Data, 0, (int)inStream.DiscHeader.Size); //write the header inStream.Copy(target, inStream.Length - inStream.DiscHeader.Size); } else { inStream.Copy(target, inStream.Length); } } else { WiiDiscHeaderSection hdr = null; using (NDisc disc = new NDisc(_log, inStream)) { foreach (IWiiDiscSection s in disc.EnumerateSections(inStream.Length)) //ctx.ImageLength { if (s is WiiDiscHeaderSection) { hdr = (WiiDiscHeaderSection)s; target.Write(hdr.Data, 0, hdr.Data.Length); //write the header pc.ReaderCheckPoint1PreWrite(null, 0); //size that we will output from this read //ctx.ImageLength } else if (s is WiiPartitionSection) { WiiPartitionSection ps = (WiiPartitionSection)s; //bool lengthChanged = inStream.CheckLength(ps.DiscOffset, ps.Header.PartitionSize); target.Write(ps.Header.Data, 0, (int)ps.Header.Size); foreach (WiiPartitionGroupSection pg in ps.Sections) { target.Write(pg.Encrypted, 0, (int)pg.Size); } } else if (s is WiiFillerSection) { WiiFillerSection fs = (WiiFillerSection)s; if (fs.Size != 0) { foreach (WiiFillerSectionItem item in ((WiiFillerSection)s).Sections) { target.Write(item.Data, 0, (int)item.Size); } } } } } } crc.Snapshot("iso"); if (inStream.IsNkit) { outputCrc = inStream.DiscHeader.ReadUInt32B(0x208); //assume source nkit crc is correct } else { outputCrc = crc.FullCrc(true); } pc.ReaderCheckPoint2Complete(crc, false, outputCrc, outputCrc, this.VerifyIsWrite, null, null); pc.ReaderCheckPoint3Complete(); } catch (Exception ex) { throw pc.SetReaderException(ex, "IsoReader.Read - Read Image"); //don't let the writer lock } }
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc) { string resultMsg = ""; if (!Settings.ConfigFileFound) { _log?.Log("!! No config file found - This is required to restore and validate images"); } NCrc crc = new NCrc(); Settings settings = ctx.Settings; List <string> addedFiles = new List <string>(); DatData data = ctx.Dats; RecoveryData rec = ctx.Recovery; List <JunkRedumpPatch> patches = rec.JunkPatches; MemorySection hdr = null; try { long junkStart = settings.JunkStartOffsets.FirstOrDefault(a => a.Id8 == inStream.Id8)?.Offset ?? 0; string forceJunkId = settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == inStream.Id8)?.JunkId; if (forceJunkId != null) { _log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", forceJunkId, inStream.Id)); //result.ImageInfo.JunkId = forceJunkId; inStream.ChangeJunk(forceJunkId); } if (junkStart != 0) { _log?.LogDetail(string.Format("Junk forced to start at 0x{0}", junkStart.ToString("X8"))); } hdr = inStream.DiscHeader; FstFileItem goodFst = rec.GcBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8 && !(inStream.Id8 == "GNHE5d0000" && a.Length == 116)); if (goodFst != null) { _log?.LogDetail(string.Format("Recovery: {0}", goodFst.Filename)); } if (goodFst == null && rec.GcNewBinFiles != null) { goodFst = rec.GcNewBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8); } if (goodFst != null) { goodFst.Populate(); } ApploaderFileItem goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc); if (goodAldr == null && rec.GcNewBinFiles != null) { goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcNewBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc); } //is it an action replay (custom hacks) if ((hdr.ReadUInt32B(0x420) == 0 && inStream.Id8 == "GNHE5d0000") || inStream.Id8 == "DTLX010000" || inStream.Id8 == "102E010007") { if (inStream.Id8 == "102E010007") { resultMsg = "Aging Disc detected - Skipping recover"; } else { resultMsg = "Datel Action Replay detected - Skipping recover"; } try { using (CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write)) { target.Write(hdr.Data, 0, (int)hdr.Size); pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read inStream.Copy(target, pc.OutputSize - hdr.Size); } _log?.LogDetail(resultMsg); } catch (Exception ex) { throw new HandledException(ex, resultMsg); } crc.Snapshot("files"); } else { //############################################################################ //# READ DISC START //########### Header (boot.bin) 0 (read by base stream already) long srcPos = hdr.Size; _log?.LogDetail(string.Format("Header Read: {0} - {1}", friendly(inStream.Id), friendly(inStream.Title))); //########### Header Info (bi2.bin) 0x440 MemorySection bi2bin = MemorySection.Read(inStream, 0x2000); srcPos += bi2bin.Size; //########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol MemorySection appldr = MemorySection.Read(inStream, Math.Min(hdr.ReadUInt32B(0x420) == 0 ? uint.MaxValue : hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - srcPos); srcPos += appldr.Size; //########### APP (main.dol) MemorySection maindol; if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424)) { maindol = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - srcPos); } else { maindol = new MemorySection(new byte[0]); } srcPos += maindol.Size; //########### FST (fst.bin) MemorySection srcFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428)); srcPos += hdr.ReadUInt32B(0x428); //############################################################################ //# CORRECT ISSUES List <FstFile> srcFiles = FileSystem.Parse(srcFst, null, inStream.Id, true)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList(); if (srcFiles == null) { throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", hdr.ReadUInt32B(0x424).ToString("X8"))); } uint crcTmp = Crc.Compute(appldr.Data, 0, Math.Min((int)appldr.Size, 0x20 + (int)(appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18)))); //adjust appldr if (goodAldr != null && crcTmp != goodAldr.Crc) { _log?.LogDetail(string.Format("Replacing appldr.bin crc {0} with Recovery appldr.bin {1}", crcTmp.ToString("X8"), goodAldr.Crc.ToString("X8"))); addedFiles.Add(goodAldr.Filename); appldr = new MemorySection(File.ReadAllBytes(goodAldr.Filename)); } //adjust main.dol if (goodFst != null) { if (goodFst.MainDolOffset != hdr.ReadUInt32B(0x420)) { //main.dol is before fst in image and after in goodfst if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424) && goodFst.MainDolOffset > goodFst.FstOffset) { _log?.LogDetail(string.Format("Skipping main.dol at 0x{0} using 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8"))); maindol = new MemorySection(new byte[0]); } else { _log?.LogDetail(string.Format("Moving main.dol address 0x{0} to 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8"))); } hdr.WriteUInt32B(0x420, (uint)goodFst.MainDolOffset); } if (goodFst.Region != (Region)bi2bin.ReadUInt32B(0x18)) { _log?.LogDetail(string.Format("Region Changed to {0} from {1}", goodFst.Region.ToString(), ((Region)bi2bin.ReadUInt32B(0x18)).ToString())); bi2bin.WriteUInt32B(0x18, (uint)goodFst.Region); } if (goodFst.MaxFst != hdr.ReadUInt32B(0x42C)) { _log?.LogDetail(string.Format("Max Fst Size changed to {0} from {1}", goodFst.MaxFst.ToString("X8"), hdr.ReadUInt32B(0x42C).ToString("X8"))); hdr.WriteUInt32B(0x42C, (uint)goodFst.MaxFst); } string newTitle = Encoding.ASCII.GetString(goodFst.Title); if (newTitle != hdr.ReadString(0x20, 0x60 - 0x20)) { _log?.LogDetail(string.Format("Title changed to '{0}' from '{1}'", newTitle.TrimEnd('\0'), hdr.ReadString(0x20, 0x60 - 0x20).TrimEnd('\0'))); hdr.Write(0x20, goodFst.Title); } } MemorySection fst = srcFst; List <FstFile> fstFiles = srcFiles; crcTmp = Crc.Compute(srcFst.Data); if (goodFst != null && crcTmp != goodFst.Crc) { _log?.LogDetail(string.Format("Replacing fst.bin crc {0} with Recovery fst {1}", crcTmp.ToString("X8"), goodFst.Crc.ToString("X8"))); addedFiles.Add(goodFst.Filename); fst = new MemorySection(goodFst.FstData); fstFiles = FileSystem.Parse(fst, null, inStream.Id, true).Files.OrderBy(a => a.Offset).ThenBy(a => a.Length).ToList(); } //adjust fst.bin if (goodFst != null && (goodFst.FstOffset != hdr.ReadUInt32B(0x424) || goodFst.FstData.Length != hdr.ReadUInt32B(0x428))) { _log?.LogDetail(string.Format("Moving / Resizing fst.bin address 0x{0} (Length {1}) to 0x{2} (Length {3})", hdr.ReadUInt32B(0x424).ToString("X8"), hdr.ReadUInt32B(0x428).ToString(), goodFst.FstOffset.ToString("X8"), goodFst.FstData.Length.ToString())); hdr.WriteUInt32B(0x424, (uint)goodFst.FstOffset); hdr.WriteUInt32B(0x428, (uint)goodFst.FstData.Length); } int c; int failCount = 0; int filesMoved = 0; int nkitJunkFiles = 0; foreach (FstFile f in srcFiles) { FstFile[] fnd = fstFiles.Where(a => a.Name == f.Name && a.Path == f.Path).ToArray(); if (fnd.Length == 0) { failCount++; _log?.LogDetail(string.Format("FST Error: No File Found - {0}/{1} (Length {2})", f.Path, f.Name, f.Length.ToString())); } else if (inStream.IsNkit && (c = fnd.Count(a => a.Length != 0 && f.Length == 0)) == 1) { nkitJunkFiles++; } else if ((c = fnd.Count(a => a.Length == f.Length)) != 1) { failCount++; _log?.LogDetail(string.Format("FST Error: File Size bad - {0}/{1} (Length {2}){3}", f.Path, f.Name, f.Length.ToString(), c <= 1 ? "" : string.Format(" {0} files found", c.ToString()))); } else if (f.DataOffset != fnd[0].DataOffset) { filesMoved++; } if (failCount >= 10) { break; } } if (failCount != 0) { if (failCount >= 10) { throw new HandledException(string.Format("{0} or more FST errors found", failCount.ToString())); } } if (filesMoved != 0) { _log?.LogDetail(string.Format("{0} file{1} of {2} will be repositioned when rebuilding this image", filesMoved.ToString(), filesMoved == 1 ? "" : "s", fstFiles.Count.ToString())); } if (nkitJunkFiles != 0) { _log?.LogDetail(string.Format("{0} file{1} of {2} will be generated from junk when rebuilding this image", nkitJunkFiles.ToString(), nkitJunkFiles == 1 ? "" : "s", fstFiles.Count.ToString())); } if (goodFst != null) { //is the data output so far correct if (!bruteForceValidHeader(ctx, goodFst, hdr, bi2bin, appldr, maindol, fst, pc)) { throw new HandledException(string.Format("Post FST Crc Failed 0x{0}", goodFst.PostFstCrc.ToString("X8"))); } } //############################################################################ //# WRITE DISC START CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write); target.Write(hdr.Data, 0, (int)hdr.Size); pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read long dstPos = hdr.Data.Length; crc.Snapshot("boot.bin"); target.Write(bi2bin.Data, 0, (int)bi2bin.Size); dstPos += bi2bin.Size; crc.Snapshot("bi2.bin"); target.Write(appldr.Data, 0, (int)appldr.Size); dstPos += appldr.Size; ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos); dstPos += Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos; crc.Snapshot("appldr.bin"); target.Write(maindol.Data, 0, (int)maindol.Size); dstPos += maindol.Size; if (goodFst != null) { long padding = goodFst.FstOffset - dstPos; ByteStream.Zeros.Copy(target, padding); dstPos += padding; } crc.Snapshot("main.dol"); target.Write(fst.Data, 0, fst.Data.Length); dstPos += fst.Size; bool firstFile = true; Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>(); crc.Snapshot("fst.bin"); //############################################################################ //# WRITE THE FILESYSTEM int fidx = -1; FstFile lastFile = new FstFile(null) { DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = fst.Size }; FstFile nextFile = fstFiles[++fidx]; FstFile cacheFile = null; long nullsPos = dstPos + 0x1c; if (nullsPos % 4 != 0) { nullsPos += 4 - (nullsPos % 4); } //########### FILES foreach (FstFile f in srcFiles) //read the files and write them out as goodFiles (possible order difference) { while ((cacheFile = cache.Keys.FirstOrDefault(a => (a.Length == nextFile.Length || (inStream.IsNkit && a.Length == 0)) && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) //write cache { bool isJunk = inStream.IsNkit && cacheFile.Length == 0 && cacheFile.Name == nextFile.Name && cacheFile.Path == nextFile.Path; using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile])) { dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, isJunk); } cache.Remove(cacheFile); lastFile = nextFile; nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null; } if (f.DataOffset - srcPos > 0) //skip src junk { inStream.Copy(ByteStream.Zeros, f.DataOffset - srcPos); srcPos += f.DataOffset - srcPos; } if (nextFile != null && (f.Length != nextFile.Length || f.Name != nextFile.Name || f.Path != nextFile.Path)) //cache file (nkit junk files are cached) { bool isNkitJunk = inStream.IsNkit && f.Length == 0 && nextFile.Length != 0 && f.Name == nextFile.Name && f.Path == nextFile.Path; byte[] cacheItem = new byte[isNkitJunk ? 8 : f.Length]; inStream.Read(cacheItem, 0, cacheItem.Length); //read the nkit junk data - details (real length and null counts) cache.Add(f, cacheItem); srcPos += cacheItem.Length; } else //copy file { dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, inStream, lastFile, nextFile, patches, inStream, junkStart, false); lastFile = nextFile; nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null; srcPos += f.Length; } } while (nextFile != null && fidx < fstFiles.Count && (cacheFile = cache.Keys.FirstOrDefault(a => a.Length == nextFile.Length && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) { using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile])) { dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, false); } cache.Remove(cacheFile); lastFile = nextFile; nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null; } writeDestGap(nullsPos, target, dstPos, pc.OutputSize - dstPos, true, patches, junkStart, inStream); crc.Snapshot("files"); } resultMsg = "MatchFail"; uint finalCrc = crc.FullCrc(true); if (ctx.Dats.RedumpData.FirstOrDefault(a => a.Crc == finalCrc) != null) { resultMsg = "Match Redump"; } if (ctx.Dats.CustomData.FirstOrDefault(a => a.Crc == finalCrc) != null) { resultMsg = "Match Custom"; } pc.ReaderCheckPoint2Complete(crc, false, finalCrc, finalCrc, VerifyIsWrite, hdr.Data, resultMsg); pc.ReaderCheckPoint3Complete(); } catch (Exception ex) { throw pc.SetReaderException(ex, "RestoreReaderGc.Read - Read and repair"); //don't let the writer lock } }
public void Write(Context ctx, Stream input, Stream output, Coordinator pc) { try { long imageSize = pc.OutputSize; string junkId; pc.WriterCheckPoint1WriteReady(out junkId); //wait until read has written the header and set the length MemorySection ms = new MemorySection(new byte[4 + 4 + 8 + 8 + 4 + 4]); int blockSize = 0x4000; int blocks = (int)(imageSize / blockSize) + (imageSize % blockSize == 0 ? 0 : 1); ms.WriteUInt32L(0x00, 0xB10BC001); ms.WriteUInt32L(0x04, 0); //insert NKIT later ms.WriteUInt64L(0x08, 0); //size place holder ms.WriteUInt64L(0x10, (ulong)imageSize); ms.WriteUInt32L(0x18, (uint)blockSize); ms.WriteUInt32L(0x1C, (uint)blocks); MemorySection pnt = new MemorySection(new byte[blocks * 8]); MemorySection hsh = new MemorySection(new byte[blocks * 4]); long offset = 0; NCrc crc = new NCrc(); CryptoStream target = new CryptoStream(output, crc, CryptoStreamMode.Write); target.Write(ms.Data, 0, (int)ms.Size); crc.Snapshot("Header"); target.Write(pnt.Data, 0, (int)pnt.Size); crc.Snapshot("Pointers"); target.Write(hsh.Data, 0, (int)hsh.Size); crc.Snapshot("Hashes"); long dataOffset = (pnt.Size + hsh.Size + ms.Size); //fso.position int rdBlk = 0; int wrBlk = 0; object rdBlkLock = new object(); object wrBlkLock = new object(); Task[] tasks = new Task[3]; ManualResetEvent mr = new ManualResetEvent(false); for (int i = 0; i < tasks.Length; i++) //4 threads { tasks[i] = (new TaskFactory()).StartNew((Object prm) => { ManualResetEvent m = (ManualResetEvent)prm; int blkIdx; byte[] rawBlk = new byte[blockSize]; byte[] buff = new byte[blockSize + 0x100]; using (MemoryStream blk = new MemoryStream(buff)) { while (true) { lock (rdBlkLock) { blkIdx = rdBlk++; if (blkIdx >= blocks) { return; //no more processing } int read = input.Read(rawBlk, 0, (int)Math.Min(imageSize - (blkIdx * blockSize), blockSize)); if (read != blockSize) { Array.Clear(rawBlk, read, blockSize - read); } } blk.Position = 0; ZlibStream zl = new ZlibStream(blk, SharpCompress.Compressors.CompressionMode.Compress, SharpCompress.Compressors.Deflate.CompressionLevel.BestCompression, Encoding.Default); zl.FlushMode = FlushType.Finish; zl.Write(rawBlk, 0, blockSize); zl.Flush(); while (blkIdx != wrBlk) { m.WaitOne(); } mr.Reset(); if (blk.Position < blockSize) { target.Write(buff, 0, (int)blk.Position); pnt.WriteUInt64L(blkIdx * 8, (ulong)(offset)); hsh.WriteUInt32L(blkIdx * 4, adler32(buff, (int)blk.Position)); offset += (int)blk.Position; } else { target.Write(rawBlk, 0, blockSize); pnt.WriteUInt64L(blkIdx * 8, (ulong)(offset) | 0x8000000000000000); hsh.WriteUInt32L(blkIdx * 4, adler32(rawBlk, blockSize)); offset += (int)blockSize; } target.Flush(); wrBlk++; mr.Set(); } } }, mr); } Task.WaitAll(tasks); crc.Snapshot("Files"); NCrc readerCrcs; uint validationCrc; pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, null, dataOffset + offset); //wait until reader has completed and get crc patches. ms.WriteUInt64L(0x08, (ulong)offset); crc.Crcs[0].PatchCrc = Crc.Compute(ms.Data); crc.Crcs[0].PatchData = ms.Data; crc.Crcs[1].PatchCrc = Crc.Compute(pnt.Data); crc.Crcs[1].PatchData = pnt.Data; crc.Crcs[2].PatchCrc = Crc.Compute(hsh.Data); crc.Crcs[2].PatchData = hsh.Data; ms.WriteUInt32B(0x04, CrcForce.Calculate(crc.FullCrc(true), output.Length, readerCrcs.FullCrc(true), 0x04, ms.ReadUInt32B(0x04))); //magic to force crc crc.Crcs[0].PatchCrc = Crc.Compute(ms.Data); pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "GCZ Written"); } catch (Exception ex) { throw pc.SetWriterException(ex, "GczWriter.Write - Compress"); } }
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc) { try { WiiDiscHeaderSection hdr = null; WiiPartitionHeaderSection pHdr = null; string lastPartitionId = null; PartitionType lastPartitionType = PartitionType.Other; NCrc crc = new NCrc(); Crc updateCrc = new Crc(); bool updateRemoved = false; string updateTmpFileName = null; string updateFileName = null; bool extractingUpdate = false; CryptoStream updateCrcStream = null; NStream updateTarget = null; CryptoStream target = null; MemorySection removedUpdateFiller = null; int preservedHashCount = 0; NkitInfo nkitDiscInfo = new NkitInfo(); long fstFileAlignment = -1; WiiPartitionSection lastPart = null; long dstPos = 0; long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength string ignoreJunkId; pc.WriterCheckPoint1WriteReady(out ignoreJunkId); //wait until read has written the header and set the length NDisc disc = new NDisc(_log, inStream); foreach (IWiiDiscSection s in disc.EnumerateSections(imageSize)) //no fixing, image should be good //ctx.ImageLength { if (s is WiiDiscHeaderSection) { hdr = (WiiDiscHeaderSection)s; hdr.Write8(0x60, 1); hdr.Write8(0x61, 1); fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == hdr.Id8)?.Item2 ?? -1; target = new CryptoStream(outStream, crc, CryptoStreamMode.Write); target.Write(hdr.Data, 0, hdr.Data.Length); //write the header nkitDiscInfo.BytesData += hdr.Size; dstPos += hdr.Size; crc.Snapshot("Disc Header"); } else if (s is WiiPartitionSection) { WiiPartitionSection ps = (WiiPartitionSection)s; WiiHashStore hashes = new WiiHashStore(ps.PartitionDataLength); ScrubManager scrub = ps.Header.ScrubManager; NkitInfo nkitPartInfo = new NkitInfo(); if (ps.Header.Type == PartitionType.Update && ctx.Settings.NkitUpdatePartitionRemoval && hdr.Partitions.Count() > 1) //only remove update if there's more partitions { updateRemoved = true; extractingUpdate = true; pHdr = ps.Header; extractingUpdate = true; updateCrcStream = new CryptoStream(ByteStream.Zeros, updateCrc, CryptoStreamMode.Write); crc.Initialize(); nkitPartInfo.BytesData += disc.WriteRecoveryPartitionData(updateCrcStream, false, ps, 0, out updateTmpFileName, out updateFileName, out updateTarget); _log?.LogDetail(string.Format("Extracted and Removed {0} Recovery Partition: [{1}]", ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4))); removedUpdateFiller = new MemorySection(new byte[0x8000]); removedUpdateFiller.Write(0, hdr.Read(0x40000, 0x100)); //backup the original partition table in case it's non standard in some way } else { target.Write(ps.Header.Data, 0, (int)ps.Header.Size); nkitDiscInfo.BytesData += ps.Header.Size; crc.Snapshot(ps.Id + " Hdr"); crc.Crcs.Last().PatchData = ps.Header.Data; dstPos += ps.Header.Size; ps.NewDiscOffset = dstPos; //long written = 0; using (StreamCircularBuffer decrypted = new StreamCircularBuffer(ps.PartitionDataLength, null, null, output => { //read decrypted partition foreach (WiiPartitionGroupSection pg in ps.Sections) { nkitPartInfo.BytesHashesData += (pg.Size / 0x8000 * 0x400); //size of input hashes if (pg.PreserveHashes()) { nkitPartInfo.BytesHashesPreservation += hashes.Preserve(pg.Offset, pg.Decrypted, pg.Size); if (++preservedHashCount >= 1500) //too many will bomb the patch caching when converting back. Something is wrong { throw pc.SetWriterException(new HandledException("Over 1500 hashes preserved, aborting as image is corrupt or poorly scrubbed.")); } } #if !DECRYPT for (int i = 0; i < pg.Size / 0x8000; i++) { output.Write(pg.Decrypted, (i * 0x8000) + 0x400, 0x7c00); } #else output.Write(pg.Decrypted, 0, (int)pg.Size); #endif } })) { long len = partitionWrite(decrypted, crc, target, ps, ctx, pc, nkitPartInfo, scrub, hashes, fstFileAlignment); ps.NewPartitionDataLength = len; dstPos += len; lastPart = ps; } } NkitFormat.LogNkitInfo(nkitPartInfo, _log, ps.Id, false); lastPartitionId = ps.Id; lastPartitionType = ps.Header.Type; } else if (s is WiiFillerSection) { WiiFillerSection fs = (WiiFillerSection)s; ScrubManager scrub = new ScrubManager(null); JunkStream junk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize); if (lastPartitionType == PartitionType.Update && updateRemoved) { //preserve the original partition table and update filename target.Write(removedUpdateFiller.Data, 0, (int)removedUpdateFiller.Size); //remove update partition by adding a 32k placeholder. Avoid having a non update partition at 0x50000 nkitDiscInfo.BytesPreservationDiscPadding += removedUpdateFiller.Size; nkitDiscInfo.BytesGaps += fs.Size; dstPos += removedUpdateFiller.Size; crc.Snapshot(string.Format("{0}Replacement Filler", string.IsNullOrEmpty(lastPartitionId) ? "" : (" " + SourceFiles.CleanseFileName(lastPartitionId).PadRight(4)))); if (extractingUpdate) { int storeType; if ((storeType = disc.WriteRecoveryPartitionFiller(updateCrcStream, junk, fs.DiscOffset, true, true, fs, updateTarget, updateTmpFileName, ref updateFileName, updateCrc, true)) != 0) { _log.LogDetail(string.Format("{0}Update recovery partition stored: {1}", storeType == 2 ? "Other " : "", updateFileName)); } extractingUpdate = false; } } else { if (fs.Size != 0) { using (StreamCircularBuffer filler = new StreamCircularBuffer(fs.Size, null, null, output => { foreach (WiiFillerSectionItem item in ((WiiFillerSection)s).Sections) { output.Write(item.Data, 0, (int)item.Size); } })) { Gap gap = new Gap(fs.Size, false); long srcPos = fs.DiscOffset; long gapLen = gap.Encode(filler, ref srcPos, lastPartitionType == PartitionType.Update ? fs.Size : 0x1cL, fs.Size, junk, scrub, target, _log); nkitDiscInfo.BytesPreservationData += gapLen; nkitDiscInfo.BytesGaps += fs.Size; dstPos += gapLen; if (lastPart != null) { lastPart.NewPartitionDataLength += gapLen; } } } //pad partition to 32k int gapLen2 = (int)(dstPos % 0x8000 == 0 ? 0 : 0x8000 - (dstPos % 0x8000)); ByteStream.Zeros.Copy(target, gapLen2); nkitDiscInfo.BytesPreservationDiscPadding += gapLen2; dstPos += gapLen2; if (lastPart != null) { lastPart.NewPartitionDataLength += gapLen2; } if (lastPart != null) { lastPart.Header.WriteUInt32B(0x2bc, (uint)(lastPart.NewPartitionDataLength / 4)); //updates the array in the crc data hdr.Partitions.First(a => a.DiscOffset == lastPart.DiscOffset).DiscOffset = (lastPart.NewDiscOffset - 0x20000); } crc.Snapshot(string.Format("{0}{1}Files+Filler", lastPartitionId ?? "", string.IsNullOrEmpty(lastPartitionId) ? "" : " ")); } } } NkitFormat.LogNkitInfo(nkitDiscInfo, _log, hdr.Id, true); foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null)) { ci.PatchCrc = Crc.Compute(ci.PatchData); } NCrc readerCrcs; uint validationCrc; pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches. if (updateRemoved && crc.Crcs.Length > 2) //freeloader wii only has update partition { hdr.RemoveUpdatePartition(crc.Crcs[2].Offset); } hdr.UpdateOffsets(); hdr.WriteString(0x200, 8, "NKIT v01"); //header and version hdr.Write8(0x60, 1); hdr.Write8(0x61, 1); hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc hdr.WriteUInt32B(0x210, (uint)(imageSize / 4L)); //ctx.ImageLength hdr.WriteUInt32B(0x218, updateCrc.Value); //Update crc - Only if removed crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); crc.Crcs[0].PatchData = hdr.Data; hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); //update with magic applied pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "NKit Written"); } catch (Exception ex) { throw pc.SetWriterException(ex, "NkitWriterWii.Write - Convert"); } }
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc) { try { long mlt = 1L; //for Wii: 4L long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength pc.WriterCheckPoint1WriteReady(out string junkId); //wait until read has written the header and set the length List <string> addedFiles = new List <string>(); NCrc crc = new NCrc(); long srcPos; long dstPos = 0; MemorySection hdr = MemorySection.Read(inStream, 0x440); string id8 = string.Concat(hdr.ReadString(0, 6), hdr.Data[6].ToString("X2"), hdr.Data[7].ToString("X2")); if (junkId == null) { junkId = ctx.Settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == id8)?.JunkId; if (junkId != null) { _log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", junkId, id8.Substring(0, 4))); } } if (junkId == null) { junkId = hdr.ReadString(0, 4); } MemorySection fst; List <JunkDiff> junkDiffs = new List <JunkDiff>(); long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt; long fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == id8)?.Item2 ?? -1; CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //############################################################################ //# READ DISC START target.Write(hdr.Data, 0, (int)hdr.Size); crc.Snapshot("hdr.bin"); inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size); crc.Snapshot("bi2.bin, appldr.bin, main.dol"); //read fst with 4 byte boundary fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4)))); target.Write(fst.Data, 0, (int)fst.Size); crc.Snapshot("fst.bin"); srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size; long nullsPos = srcPos + 0x1c; dstPos = srcPos; //create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample) JunkStream js = new JunkStream(junkId, hdr.Read8(6), NStream.FullSizeGameCube); List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, imageSize, hdr, fst, true, fstFileAlignment, out string error); //Size isn't important for writing //result.ImageInfo.IsoSize NkitInfo nkitInfo = new NkitInfo { BytesData = srcPos, BytesGaps = 0, BytesJunkFiles = 0, BytesPreservationData = 0, BytesPreservationDiscPadding = 0 }; ScrubManager scrub = new ScrubManager(); if (conFiles == null) { if (error != null) { _log?.LogDetail(error); } ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize { FstFile = new FstFile(null) { DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size }, }; NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log); } else { //############################################################################ //# WRITE THE FILESYSTEM NkitFormat.NkitWriteFileSystem(ctx, nkitInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out List <ConvertFile> missing, scrub, imageSize, _log); if (missing.Count != 0) { _log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s")); foreach (ConvertFile cf in missing) { _log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length)); } } } if (dstPos % 0x800 != 0) { long l = 0x800 - (dstPos % 0x800); ByteStream.Zeros.Copy(target, l); dstPos += l; nkitInfo.BytesPreservationDiscPadding += l; } crc.Snapshot("files"); NkitFormat.LogNkitInfo(nkitInfo, _log, hdr.ReadString(0, 4), true); pc.WriterCheckPoint2Complete(out NCrc readerCrcs, out uint validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches. hdr.WriteUInt32B(0x420, (uint)mainDolAddr); hdr.WriteString(0x200, 8, "NKIT v01"); //header and version hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc hdr.WriteUInt32B(0x210, (uint)imageSize); //result.ImageInfo.IsoSize); hdr.WriteString(0x214, 4, hdr.ReadString(0, 4) != junkId ? junkId : "\0\0\0\0"); crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); crc.Crcs[0].PatchData = hdr.Data; crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data); crc.Crcs[2].PatchData = fst.Data; hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, "NKit Written"); } catch (Exception ex) { throw pc.SetWriterException(ex, "NkitWriterGc.Write - Convert"); } }
public ExtractResult ExtractRecoveryFilesGc() { MemorySection hdr = NStream.DiscHeader; MemorySection bi2bin; List <ExtractRecoveryResult> result = new List <ExtractRecoveryResult>(); NCrc crc = new NCrc(); uint appLdrCrc = 0; MemorySection fst = null; uint fstCrc = 0; string fn; string tmpFullName; Log?.Log(string.Format("Processing: {0}", SourceFileName)); int storeType; using (CryptoStream bw = new CryptoStream(ByteStream.Zeros, crc, CryptoStreamMode.Write)) { bw.Write(hdr.Data, 0, (int)hdr.Size); bi2bin = MemorySection.Read(NStream, 0x2000); bw.Write(bi2bin.Data, 0, (int)bi2bin.Size); MemorySection aplHdr = MemorySection.Read(NStream, 0x20); byte[] appLdr = new byte[0x20 + aplHdr.ReadUInt32B(0x14) + aplHdr.ReadUInt32B(0x18)]; Array.Copy(aplHdr.Data, appLdr, (int)aplHdr.Size); NStream.Read(appLdr, (int)aplHdr.Size, appLdr.Length - (int)aplHdr.Size); appLdrCrc = Crc.Compute(appLdr); fn = string.Format("appldr[{0}][{1}].bin", aplHdr.ReadString(0, 10).Replace("/", ""), appLdrCrc.ToString("X8")); Log?.Log(string.Format(" 1 of 2 - Extracting appldr.bin Recovery File: {0}", fn)); tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + appLdrCrc.ToString("X8"); if (File.Exists(tmpFullName)) { File.Delete(tmpFullName); } File.WriteAllBytes(tmpFullName, appLdr); if ((storeType = storeRecoveryFile(this.Settings.RedumpAppldrCrcs.Contains(appLdrCrc), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0) { result.Add(new ExtractRecoveryResult() { FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true }); } bw.Write(appLdr, 0, appLdr.Length); //add to fullcrc NStream.Copy(bw, hdr.ReadUInt32B(0x424) - (0x2440 + appLdr.Length)); byte[] fstData = new byte[(int)((4 * 4) + (0x60 - 0x20) + hdr.ReadUInt32B(0x428))]; fst = new MemorySection(fstData); //info: dolAddr, fstAddr, maxfst, region, title, fst NStream.Read(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428)); fst.WriteUInt32B(0x00, hdr.ReadUInt32B(0x420)); //dol fst.WriteUInt32B(0x04, hdr.ReadUInt32B(0x424)); //fstAddr fst.WriteUInt32B(0x08, hdr.ReadUInt32B(0x42C)); //maxfst fst.WriteUInt32B(0x0c, bi2bin.ReadUInt32B(0x18)); //region fst.Write(0x10, hdr.Read(0x20, 0x60 - 0x20)); //title fstCrc = Crc.Compute(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428)); //fst bw.Write(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428)); crc.Snapshot("crc"); } //fst checksums are postfst fn = string.Format("fst[{0}][{1}][{2}][{3}].bin", NStream.Id8, appLdrCrc.ToString("X8"), fstCrc.ToString("X8"), crc.FullCrc().ToString("X8")); Log?.Log(string.Format(" 2 of 2 - Extracting fst.bin Recovery File: {0}", fn)); tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + crc.FullCrc().ToString("X8"); if (File.Exists(tmpFullName)) { File.Delete(tmpFullName); } File.WriteAllBytes(tmpFullName, fst.Data); if ((storeType = storeRecoveryFile(this.Settings.RedumpFstCrcs.Contains(crc.FullCrc()), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0) { result.Add(new ExtractRecoveryResult() { FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true }); } return(createExtractResult((Region)bi2bin.Read8(0x18), result.ToArray())); }
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc) { try { DatData data = ctx.Dats; List <string> addedFiles = new List <string>(); DateTime dt = DateTime.Now; MemorySection hdr = inStream.DiscHeader; string idVer = hdr.ReadString(0x200, 8); if (idVer != "NKIT v01") { throw new Exception(string.Format("{0} not supported by this version", idVer)); } bool isNkit = idVer.StartsWith("NKIT"); uint nkitCrc = hdr.ReadUInt32B(0x208); uint imageSize = hdr.ReadUInt32B(0x210); string junkId = hdr.ReadString(0x214, 4); if (junkId != "\0\0\0\0") { inStream.ChangeJunk(junkId); } hdr.WriteUInt32B(0x200, 0); hdr.WriteUInt32B(0x204, 0); hdr.WriteUInt32B(0x208, 0); hdr.WriteUInt32B(0x20C, 0); hdr.WriteUInt32B(0x210, 0); hdr.WriteUInt32B(0x214, 0); hdr.WriteUInt32B(0x218, 0); MemorySection fst; long mainDolAddr = hdr.ReadUInt32B(0x420); NCrc crc = new NCrc(); NStream target = new NStream(new CryptoStream(outStream, crc, CryptoStreamMode.Write)); //############################################################################ //# READ DISC START MemorySection hdrToFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - hdr.Size); fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) + (hdr.ReadUInt32B(0x428) % 4 == 0 ? 0 : 4 - (hdr.ReadUInt32B(0x428) % 4))); long srcPos = hdr.ReadUInt32B(0x424) + fst.Size; //############################################################################ //# WRITE DISC START target.Write(hdr.Data, 0, (int)hdr.Size); pc.ReaderCheckPoint1PreWrite(junkId, hdr.ReadUInt32B(0x208)); //size that we will output from this read crc.Snapshot("hdr.bin"); target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read crc.Snapshot("bi2.bin, appldr.bin, main.dol"); target.Write(fst.Data, 0, fst.Data.Length); crc.Snapshot("fst.bin"); hdrToFst = null; //let this be collected if needed long dstPos = hdr.ReadUInt32B(0x424) + fst.Size; long nullsPos = dstPos + 0x1c; string error; List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, inStream.Length, hdr, fst, true, -1, out error); //result.ImageInfo.IsoSize if (conFiles == null) { if (error != null) { _log?.LogDetail(error); } ConvertFile cf = new ConvertFile(inStream.Length - srcPos, true) //result.ImageInfo.IsoSize { FstFile = new FstFile(null) { DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size }, }; dstPos += writeGap(cf, ref nullsPos, ref srcPos, dstPos, inStream, target, true); } else { //########### FILES bool firstFile = true; for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference { ConvertFile f = conFiles[i]; FstFile ff = f.FstFile; if (!firstFile) //fst already written { //Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), dstPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (dstPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name)); if (srcPos < ff.DataOffset) { inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc srcPos += ff.DataOffset - srcPos; } //write file if (ff.DataOffset == mainDolAddr) { hdr.WriteUInt32B(0x420, (uint)dstPos); } fst.WriteUInt32B(ff.OffsetInFstFile, (uint)dstPos); dstPos += copyFile(f, ref nullsPos, ref srcPos, dstPos, imageSize, inStream, target); } if (dstPos < imageSize) { dstPos += writeGap(f, ref nullsPos, ref srcPos, dstPos, inStream, target, i == 0 || i == conFiles.Count - 1); if (!firstFile) { fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)ff.Length); } } firstFile = false; } } crc.Snapshot("files"); crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); crc.Crcs[0].PatchData = hdr.Data; crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data); crc.Crcs[2].PatchData = fst.Data; if (imageSize != dstPos) { throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString())); } pc.ReaderCheckPoint2Complete(crc, false, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid"); pc.ReaderCheckPoint3Complete(); } catch (Exception ex) { throw pc.SetReaderException(ex, "NkitReaderGc.Read - Read and convert"); //don't let the writer lock } }
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc) { if (!Settings.ConfigFileFound) { _log?.Log("!! No config file found - This is required to restore and validate images"); } bool truncatedRvtr = false; bool write = !(outStream is ByteStream || outStream == Stream.Null); //ProgressResult result = ctx.Result; byte[] origHeader = null; byte[] dataHeader = new byte[256]; uint headerCrc = 0; uint dataHeaderCrc = 0; //bool fstMissingWithH3Error = false; NCrc crc = new NCrc(); CryptoStream target = null; int h3Errors = 0; List <string> requiredUpdateFiles = new List <string>(); try { //long progress = 0; string lastPartitionId = ""; bool generateUpdateFiller = false; bool generateOtherFiller = true; bool forceFillerJunk = false; NDisc disc = new NDisc(_log, inStream); foreach (IWiiDiscSection s in disc.EnumerateSectionsFix(generateUpdateFiller, generateOtherFiller, forceFillerJunk)) { if (s is WiiDiscHeaderSection) { _hdr = (WiiDiscHeaderSection)s; origHeader = (byte[])_hdr.Data.Clone(); target = new CryptoStream(outStream, crc, CryptoStreamMode.Write); target.Write(_hdr.Data, 0, _hdr.Data.Length); //write the header applyPartitionTableFixes(inStream, requiredUpdateFiles, ctx.Settings, ctx.Recovery); headerCrc = Crc.Compute(_hdr.Data); crc.Snapshot("Header"); pc.ReaderCheckPoint1PreWrite(null, 0); //size that we will output from this read } else if (s is WiiPartitionSection) { WiiPartitionSection ps = (WiiPartitionSection)s; if (!truncatedRvtr && ps.Header.IsRvtR && inStream.RecoverySize == NStream.FullSizeWii5) { _log.LogDetail(string.Format("Truncated RVT-R image detected. Pad it with 00 to {0} bytes for NKit to handle it properly", NStream.FullSizeWiiRvtr.ToString())); truncatedRvtr = true; } if (applyFixes(ps.Header, inStream)) { _hdr.UpdateRepair(); headerCrc = Crc.Compute(_hdr.Data); //recalculate } if (ps.Header.Type == PartitionType.Data) { if (applyDataPartitionFixes(ps.Header)) { headerCrc = Crc.Compute(_hdr.Data); //recalculate } Array.Copy(ps.Header.Data, dataHeader, dataHeader.Length); } target.Write(ps.Header.Data, 0, (int)ps.Header.Size); foreach (WiiPartitionGroupSection pg in ps.Sections) { if (ps.Header.Type == PartitionType.Data && pg.DiscOffset == ps.Header.DiscOffset + ps.Header.Size) { Array.Copy(pg.Decrypted, 0x400, dataHeader, 0, dataHeader.Length); //copy out datapartition header (256 bytes) } pg.Unscrub(ctx.Recovery.JunkPatches); h3Errors += pg.H3Errors; target.Write(pg.Encrypted, 0, (int)pg.Size); } if (h3Errors != 0) { _log?.LogDetail(string.Format("{0} unrecoverable group errors, this image will now be corrupted due to failed unscrubbing attempts!", h3Errors.ToString())); } lastPartitionId = ps.Id; } else if (s is WiiFillerSection) { WiiFillerSection fs = (WiiFillerSection)s; if (fs.Size != 0) { foreach (WiiFillerSectionItem item in ((WiiFillerSection)s).Sections) { target.Write(item.Data, 0, (int)item.Size); } } crc.Snapshot(((WiiFillerSection)s).DiscOffset == 0x50000 ? "[Update Filler]" : lastPartitionId); } } } catch (Exception ex) { throw pc.SetReaderException(ex, "RestoreReaderWii.Restore - Read and repair"); } try { Tuple <string, string, string, uint>[] allParts = ctx.Recovery.WiiUPartsData.Union(ctx.Recovery.WiiUOtherPartsData).ToArray(); uint[] uniqueCrcs = allParts.Select(a => a.Item4).Union(ctx.Settings.RedumpUpdateCrcs.Where(a => !allParts.Any(b => a == b.Item4))).ToArray(); //create a data header based on the modified header byte[] dataHdr = dataHeader; dataHeader = (byte[])_hdr.Data.Clone(); Array.Copy(dataHdr, dataHeader, dataHdr.Length); Array.Clear(dataHeader, 0x60, 2); dataHeaderCrc = Crc.Compute(dataHeader); bool isCustom = false; bool updatePartitionMissing = false; SortedList <uint, bool> checkCrcs = new SortedList <uint, bool>(); foreach (RedumpEntry r in ctx.Dats.RedumpData) { checkCrcs.Add(r.Crc, true); } foreach (RedumpEntry r in ctx.Dats.CustomData.Where(a => !checkCrcs.ContainsKey(a.Crc))) { checkCrcs.Add(r.Crc, false); } HeaderBruteForcer crcMtch = new HeaderBruteForcer(uniqueCrcs, checkCrcs, ctx.Settings.RedumpRegionData, _hdr.Data, dataHeader); BruteForceCrcResult bfMatch = crcMtch.Match(crc.Crcs); string updateFilename = allParts?.FirstOrDefault(a => a.Item4 == (bfMatch.UpdateChanged ? bfMatch.UpdateCrc : crc.Crcs[1].Value))?.Item1; updatePartitionMissing = bfMatch.UpdateChanged && !allParts.Any(a => a.Item4 == bfMatch.UpdateCrc); //matched, but update crc not an update partition if (bfMatch.HeaderChanged) { crc.Crcs[0].PatchCrc = bfMatch.HeaderCrc; crc.Crcs[0].PatchData = bfMatch.Header; if (bfMatch.RegionChanged) { _log.LogDetail(bfMatch.Region != bfMatch.OriginalRegion ? string.Format("Region changed from {0} to {1}", ((Region)bfMatch.OriginalRegion).ToString(), ((Region)bfMatch.Region).ToString()) : string.Format("Region age ratings changed for {0} region", ((Region)bfMatch.Region).ToString())); } } bool isRecoverable = false; if (bfMatch.UpdateChanged) { if (!updatePartitionMissing) { _log?.LogDetail(string.Format("Matched recovery update partition: {0}", updateFilename ?? "")); crc.Crcs[1].Name = updateFilename ?? string.Format("[UNKNOWN {0}]", bfMatch.UpdateCrc.ToString("X8")); crc.Crcs[1].PatchCrc = bfMatch.UpdateCrc; crc.Crcs[1].PatchFile = Path.Combine(ctx.Recovery.WiiUPartsData.Any(a => a.Item4 == bfMatch.UpdateCrc) ? ctx.Settings.RecoveryFilesPath : ctx.Settings.OtherRecoveryFilesPath, updateFilename); } else { _log?.LogDetail(string.Format("Missing update recovery partition file *_{0}", bfMatch.UpdateCrc.ToString("X8"))); crc.Crcs[1].Name = "Missing Recovery Partition File"; crc.Crcs[1].PatchCrc = bfMatch.UpdateCrc; crc.Crcs[1].PatchFile = null; isRecoverable = true; } } else if (!string.IsNullOrEmpty(updateFilename)) { crc.Crcs[1].Name += string.Format(" [Matches {0}{1}]", updateFilename, isRecoverable ? " (Recoverable)" : ""); } string resultMsg = "MatchFail"; if (bfMatch.MatchedCrc != 0) { if (updatePartitionMissing) { resultMsg = string.Format("Match {0} (Recoverable: missing update partition {1})", isCustom ? "Custom" : "Redump", bfMatch.UpdateCrc.ToString("X8")); } else { resultMsg = string.Format("Match {0}", isCustom ? "Custom" : "Redump"); } } pc.ReaderCheckPoint2Complete(crc, isRecoverable, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, bfMatch?.Header ?? _hdr.Data, resultMsg); pc.ReaderCheckPoint3Complete(); } catch (Exception ex) { throw pc.SetReaderException(ex, "RestoreReaderWii.Read - Read and repair"); //don't let the writer lock } }
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc) { try { WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)inStream.DiscHeader; string idVer = hdr.ReadString(0x200, 8); if (idVer != "NKIT v01") { throw new Exception(string.Format("{0} not supported by this version", idVer)); } bool isNkit = idVer.StartsWith("NKIT"); uint nkitCrc = hdr.ReadUInt32B(0x208); long imageSize = hdr.ReadUInt32B(0x210) * 4L; string junkId = hdr.ReadString(0x214, 4); uint updatePartitionCrc = hdr.ReadUInt32B(0x218); MemorySection updateRemovedFiller = null; long discOffset = 0; List <NkitPartitionPatchInfo> patchInfos = new List <NkitPartitionPatchInfo>(); discOffset += hdr.Size; string lastPartitionId = null; PartitionType lastPartitionType = PartitionType.Other; NCrc crc = new NCrc(); long dstPos = 0; long srcPos = hdr.Size; ScrubManager scrubFiller = new ScrubManager(null); bool isRecoverable = false; using (NDisc disc = new NDisc(_log, inStream)) { hdr.WriteUInt32B(0x200, 0); hdr.WriteUInt32B(0x204, 0); hdr.WriteUInt32B(0x208, 0); hdr.WriteUInt32B(0x20C, 0); hdr.WriteUInt32B(0x210, 0); hdr.WriteUInt32B(0x214, 0); hdr.WriteUInt32B(0x218, 0); hdr.Write8(0x60, 0); hdr.Write8(0x61, 0); CryptoStream crcStream = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //wrap to calculate crc crcStream.Write(hdr.Data, 0, hdr.Data.Length); //write the header pc.ReaderCheckPoint1PreWrite(null, nkitCrc); //size that will be output from this read dstPos += hdr.Size; crc.Snapshot("Disc Header"); foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted { if (updatePartitionCrc != 0 && srcPos == hdr.Size) //write update partition out { updateRemovedFiller = MemorySection.Read(inStream, hdr.Partitions.First().DiscOffset - srcPos); srcPos += updateRemovedFiller.Size; WiiPartitionInfo firstPart = WiiDiscHeaderSection.CreatePartitionInfos(updateRemovedFiller, 0)?.FirstOrDefault(a => a.Type != PartitionType.Update); string updateFileName = RecoveryData.GetUpdatePartition(ctx.Settings, updatePartitionCrc); if (updateFileName != null) { using (FileStream pf = File.OpenRead(updateFileName)) { pf.Copy(crcStream, pf.Length); dstPos += pf.Length; } } else { string msg = string.Format("!! Update partition *_{0} missing - Adding filler. It may be Recoverable", updatePartitionCrc.ToString("X8")); isRecoverable = true; _log?.LogDetail(msg); //throw pc.SetReaderException(new HandledException("Failed to convert: " + msg)); } ByteStream.Zeros.Copy(crcStream, firstPart.DiscOffset - dstPos); //fill full gap dstPos += firstPart.DiscOffset - dstPos; } NkitPartitionPatchInfo patchInfo = new NkitPartitionPatchInfo() { HashGroups = new Dictionary <long, MemorySection>() }; patchInfos.Add(patchInfo); if (part.DiscOffset > srcPos) { dstPos += writeFiller(ref srcPos, dstPos, dstPos + 0x1cL, inStream, crcStream, new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize), scrubFiller); inStream.Copy(ByteStream.Zeros, part.DiscOffset - srcPos); //padded to 0x8000 srcPos += part.DiscOffset - srcPos; } part.DiscOffset = dstPos; //restore the original position patchInfo.DiscOffset = dstPos; patchInfo.PartitionHeader = MemorySection.Read(inStream, 0x20000); srcPos += patchInfo.PartitionHeader.Size; long size = patchInfo.PartitionHeader.ReadUInt32B(0x2bc) * 4L; LongRef origSize = new LongRef() { Value = 0 }; WiiPartitionGroupSection wp = null; WiiPartitionHeaderSection wh = new WiiPartitionHeaderSection(hdr, null, part.DiscOffset, patchInfo.PartitionHeader.Data, patchInfo.PartitionHeader.Data.Length); MemorySection ph = new MemorySection(new byte[0x8000 * 64]); long remaining = long.MaxValue;//set after first block read int groupIndex = 0; WiiHashStore hashes = new WiiHashStore(); patchInfo.ScrubManager = wh.ScrubManager; bool patchBlock = false; StreamCircularBuffer prtStream = null; try { using (prtStream = new StreamCircularBuffer(0, null, null, output => srcPos += partitionStreamWrite(origSize, inStream, output, size, ctx.Dats, patchInfo, hashes, pc))) { int gs = 0; int ge = 0; int i = 0; MemoryStream patchBlocks = null; while (remaining > 0) { int blocks = (int)Math.Min(64L, remaining / 0x7c00); for (int b = 0; b < blocks; b++) { prtStream.Read(ph.Data, (b * 0x8000) + 0x400, 0x7c00); //load aligned with no hashes if (remaining == long.MaxValue) //first loop { remaining = origSize.Value; if (ph.ReadString(0x400 + 0, 4) == "\0\0\0\0") { gs = -1; ge = -1; blocks = (int)Math.Min(64L, remaining / 0x7c00); lastPartitionId = ph.ReadString(0x400 + 0, 4); patchInfo.PartitionHeader.WriteUInt32B(0x2bc, (uint)(NStream.DataToHashedLen(origSize.Value) / 4)); //restore real size crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length); dstPos += patchInfo.PartitionHeader.Size; } else { gs = (int)((ph.ReadUInt32B(0x400 + 0x424) * 4L) / (0x7c00L * 64)); ge = (int)(((ph.ReadUInt32B(0x400 + 0x424) * 4L) + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) / (0x7c00L * 64)); if ((int)((part.DiscOffset + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) % (0x7c00L * 64)) == 0) { ge--; //don't load the next group if the data will end on the last byte } blocks = (int)Math.Min(64L, remaining / 0x7c00); lastPartitionId = ph.ReadString(0x400 + 0, 4); patchInfo.PartitionHeader.WriteUInt32B(0x2bc, ph.ReadUInt32B(0x400 + 0x210)); //restore real size crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length); dstPos += patchInfo.PartitionHeader.Size; ph.WriteUInt32B(0x400 + 0x200, 0); ph.WriteUInt32B(0x400 + 0x204, 0); ph.WriteUInt32B(0x400 + 0x208, 0); ph.WriteUInt32B(0x400 + 0x20C, 0); ph.WriteUInt32B(0x400 + 0x210, 0); ph.WriteUInt32B(0x400 + 0x214, 0); ph.WriteUInt32B(0x400 + 0x218, 0); } wp = new WiiPartitionGroupSection(hdr, wh, ph.Data, part.DiscOffset, blocks * 0x8000, false); wh.Initialise(wp, origSize.Value); } } if (blocks < 64) { Array.Clear(ph.Data, blocks * 0x8000, ph.Data.Length - (blocks * 0x8000)); //clear remaining blocks } wp.Populate(groupIndex, ph.Data, dstPos, blocks * 0x8000); int scrubbed = 0; for (int bi = 0; bi < blocks; bi++) { wp.MarkBlockDirty(bi); byte byt; if (patchInfo.ScrubManager.IsBlockScrubbedScanMode(wp.Offset + (bi * 0x8000), out byt)) { wp.SetScrubbed(bi, byt); scrubbed++; } } bool isFstBlocks = i >= gs && i <= ge; bool reqHashes = hashes.IsPreserved(wp.Offset); //test with 0 partition based offset repairBlocks(wp, scrubbed, blocks, false, isFstBlocks); //only test if the hashes aren't preserved (only preserved for scrubbed/customs) if (reqHashes) //store with disc based offset { patchInfo.HashGroups.Add(wp.Offset + part.DiscOffset + patchInfo.PartitionHeader.Size, new MemorySection((byte[])wp.Decrypted.Clone())); //fetch the stored hashed that couldn't be recreated } groupIndex++; bool inFstArea = i >= gs && i <= ge; if (!patchBlock && (gs == i || reqHashes)) { patchBlocks = new MemoryStream(); crc.Snapshot(lastPartitionId + " Data"); patchBlock = true; } else if (patchBlock && !inFstArea && !reqHashes) { crc.Snapshot(lastPartitionId + " Patch"); crc.Crcs.Last().PatchData = patchBlocks.ToArray(); patchBlocks.Dispose(); patchBlock = false; } #if DECRYPT outStream.Write(wp.Decrypted, 0, blocks * 0x8000); if (i >= gs && i <= ge) { fstBlocks.Write(wp.Decrypted, 0, blocks * 0x8000); } #else crcStream.Write(wp.Encrypted, 0, blocks * 0x8000); if (patchBlock) { patchBlocks.Write(wp.Encrypted, 0, blocks * 0x8000); } #endif remaining -= (blocks * 0x7c00); dstPos += (blocks * 0x8000); i++; } if (patchBlock) { crc.Snapshot(lastPartitionId + " Patch"); crc.Crcs.Last().PatchData = patchBlocks.ToArray(); patchBlocks.Dispose(); } } if (origSize.Value != prtStream.WriterPosition) { throw pc.SetReaderException(new HandledException("Partition read did not write the full amount to the circular buffer before completing")); } } catch (Exception ex) { if (prtStream?.WriterException != null) { throw pc.SetReaderException(prtStream.WriterException, "Failed reading filesystem"); } throw pc.SetReaderException(ex, "Failed converting the filesystem");; //writer exception } srcPos += hashes.ReadPatchData(part.DiscOffset + patchInfo.PartitionHeader.Size, patchInfo.HashGroups, inStream); //read hash patches lastPartitionType = part.Type; } if (srcPos < inStream.Length) { JunkStream partJunk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize); dstPos += writeFiller(ref srcPos, dstPos, lastPartitionType == PartitionType.Update ? imageSize : dstPos + 0x1cL, inStream, crcStream, partJunk, scrubFiller); } } crc.Snapshot("End"); if (updatePartitionCrc != 0) { hdr.Write((int)WiiDiscHeaderSection.PartitionTableOffset, updateRemovedFiller.Data, 0, (int)WiiDiscHeaderSection.PartitionTableLength); //restore the exact partition table if update was removed } else { hdr.UpdateOffsets(); //just update the table with the new offsets } crc.Crcs[0].PatchData = hdr.Data; foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null)) { NkitPartitionPatchInfo patchInfo = patchInfos.FirstOrDefault(a => ci.Offset >= a.DiscOffset + a.PartitionHeader.Size && ci.Offset < a.DiscOffset + a.PartitionHeader.Size + a.Size); if (patchInfo != null) { patchGroups(patchInfo, hdr, ci.Offset, ci.PatchData); } ci.PatchCrc = Crc.Compute(ci.PatchData); } if (imageSize != dstPos) { throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString())); } pc.ReaderCheckPoint2Complete(crc, isRecoverable, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid"); pc.ReaderCheckPoint3Complete(); } catch (Exception ex) { throw pc.SetReaderException(ex, "NkitReaderWii.Read - Read and convert"); } }