private int blockCompare(long nulls, byte[] junk, long junkLength, ScrubManager scrub, byte[] buff, int bStart, int bEnd, long srcPos, bool isGapStart, out byte?scrubByte) { scrubByte = null; int bTst = bStart; bool scrubDecrypted = false; //assume 00 are junk when outside of the range where junk is generated. For Wii we must be sure it's not decrypted 00 if (bTst < bEnd) { byte scrubB; bTst += scrub.IsScrubbed(srcPos + bStart, buff, bTst, bEnd - bStart, out scrubB, out scrubDecrypted); if (bTst == bEnd) { scrubByte = scrubB; } } if (bTst < bStart + 0x20) //arbitrary, but larger than 0x1c + 3 padding which can be nulls { int leadingNulls = (int)(isGapStart && bStart == 0 ? nulls : 0); for (bTst = bStart; bTst < bEnd; bTst++) { if (leadingNulls-- > 0) { if (buff[bTst] != 0) { break; } } else if (buff[bTst] != junk[bTst]) { break; } } } if (bTst == bEnd) { if (scrubByte == null || (scrubByte == 0x00 && (bTst - bStart <= nulls || (!scrubDecrypted && srcPos + bStart >= junkLength)))) //junk or leading nulls OR junk is requested after the partition length { return(0); } else { return(1); //scrubbed } } else { return(2); //preserve (also data that's part scrubbed part junk in a 256 byte block) } }
public long Encode(Stream s, ref long srcPos, long nulls, long gapLength, JunkStream junk, ScrubManager scrub, Stream output, ILog log) { int read = 0; long written = 0; _headerWritten = false; junk.Position = srcPos; if (gapLength != 0) { int[] results = new int[0x400]; byte[] scrubBytes = new byte[results.Length]; byte[] buff = new byte[Math.Min(gapLength, Gap.BlockSize * results.Length)]; byte[] jbuff = new byte[buff.Length]; //cache some junk up front so we can thread the block comparisons long start = srcPos; do { int req = (int)Math.Min(buff.Length, gapLength - Length); Task.WaitAll( Task.Run(() => read = s.Read(buff, 0, req)), Task.Run(() => junk.Read(jbuff, 0, req)) ); int blocks = (int)(read / Gap.BlockSize) + (int)Math.Min(1, read % Gap.BlockSize); long pos = srcPos; Parallel.For(0, blocks, bb => { if ((results[bb] = blockCompare(nulls, jbuff, junk.JunkLength, scrub, buff, (int)Gap.BlockSize * bb, (int)Math.Min(Gap.BlockSize * (bb + 1), read), pos, start == pos, out byte?scrubByte)) == 1) { scrubBytes[bb] = scrubByte.Value; } }); for (int bb = 0; bb < blocks; bb++) { if (results[bb] == 0) { Set(); //junk } else if (results[bb] == 1) { Set(scrubBytes[bb]); //scrubbed //log?.LogDebug(string.Format(">>> Scrubbed Written: {0} : {1}", srcPos.ToString("X8"), bb.ToString())); } else { //log?.LogDebug(string.Format(">>> NonJunk Written: {0} : {1}", srcPos.ToString("X8"), bb.ToString())); written += Set(buff, (int)Gap.BlockSize * bb, (int)Math.Min(Gap.BlockSize * (bb + 1), read), output); //preserve (also data that's part scrubbed part junk in a 256 byte block) } } srcPos += read; }while (Length < gapLength && read != 0); //sum of processed gap blocks < exact gap }
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc) { try { WiiDiscHeaderSection hdr = null; WiiPartitionHeaderSection pHdr = null; string lastPartitionId = null; PartitionType lastPartitionType = PartitionType.Other; NCrc crc = new NCrc(); Crc updateCrc = new Crc(); bool updateRemoved = false; string updateTmpFileName = null; string updateFileName = null; bool extractingUpdate = false; CryptoStream updateCrcStream = null; NStream updateTarget = null; CryptoStream target = null; MemorySection removedUpdateFiller = null; int preservedHashCount = 0; NkitInfo nkitDiscInfo = new NkitInfo(); long fstFileAlignment = -1; WiiPartitionSection lastPart = null; long dstPos = 0; long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength string ignoreJunkId; pc.WriterCheckPoint1WriteReady(out ignoreJunkId); //wait until read has written the header and set the length NDisc disc = new NDisc(_log, inStream); foreach (IWiiDiscSection s in disc.EnumerateSections(imageSize)) //no fixing, image should be good //ctx.ImageLength { if (s is WiiDiscHeaderSection) { hdr = (WiiDiscHeaderSection)s; hdr.Write8(0x60, 1); hdr.Write8(0x61, 1); fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == hdr.Id8)?.Item2 ?? -1; target = new CryptoStream(outStream, crc, CryptoStreamMode.Write); target.Write(hdr.Data, 0, hdr.Data.Length); //write the header nkitDiscInfo.BytesData += hdr.Size; dstPos += hdr.Size; crc.Snapshot("Disc Header"); } else if (s is WiiPartitionSection) { WiiPartitionSection ps = (WiiPartitionSection)s; WiiHashStore hashes = new WiiHashStore(ps.PartitionDataLength); ScrubManager scrub = ps.Header.ScrubManager; NkitInfo nkitPartInfo = new NkitInfo(); if (ps.Header.Type == PartitionType.Update && ctx.Settings.NkitUpdatePartitionRemoval && hdr.Partitions.Count() > 1) //only remove update if there's more partitions { updateRemoved = true; extractingUpdate = true; pHdr = ps.Header; extractingUpdate = true; updateCrcStream = new CryptoStream(ByteStream.Zeros, updateCrc, CryptoStreamMode.Write); crc.Initialize(); nkitPartInfo.BytesData += disc.WriteRecoveryPartitionData(updateCrcStream, false, ps, 0, out updateTmpFileName, out updateFileName, out updateTarget); _log?.LogDetail(string.Format("Extracted and Removed {0} Recovery Partition: [{1}]", ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4))); removedUpdateFiller = new MemorySection(new byte[0x8000]); removedUpdateFiller.Write(0, hdr.Read(0x40000, 0x100)); //backup the original partition table in case it's non standard in some way } else { target.Write(ps.Header.Data, 0, (int)ps.Header.Size); nkitDiscInfo.BytesData += ps.Header.Size; crc.Snapshot(ps.Id + " Hdr"); crc.Crcs.Last().PatchData = ps.Header.Data; dstPos += ps.Header.Size; ps.NewDiscOffset = dstPos; //long written = 0; using (StreamCircularBuffer decrypted = new StreamCircularBuffer(ps.PartitionDataLength, null, null, output => { //read decrypted partition foreach (WiiPartitionGroupSection pg in ps.Sections) { nkitPartInfo.BytesHashesData += (pg.Size / 0x8000 * 0x400); //size of input hashes if (pg.PreserveHashes()) { nkitPartInfo.BytesHashesPreservation += hashes.Preserve(pg.Offset, pg.Decrypted, pg.Size); if (++preservedHashCount >= 1500) //too many will bomb the patch caching when converting back. Something is wrong { throw pc.SetWriterException(new HandledException("Over 1500 hashes preserved, aborting as image is corrupt or poorly scrubbed.")); } } #if !DECRYPT for (int i = 0; i < pg.Size / 0x8000; i++) { output.Write(pg.Decrypted, (i * 0x8000) + 0x400, 0x7c00); } #else output.Write(pg.Decrypted, 0, (int)pg.Size); #endif } })) { long len = partitionWrite(decrypted, crc, target, ps, ctx, pc, nkitPartInfo, scrub, hashes, fstFileAlignment); ps.NewPartitionDataLength = len; dstPos += len; lastPart = ps; } } NkitFormat.LogNkitInfo(nkitPartInfo, _log, ps.Id, false); lastPartitionId = ps.Id; lastPartitionType = ps.Header.Type; } else if (s is WiiFillerSection) { WiiFillerSection fs = (WiiFillerSection)s; ScrubManager scrub = new ScrubManager(null); JunkStream junk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize); if (lastPartitionType == PartitionType.Update && updateRemoved) { //preserve the original partition table and update filename target.Write(removedUpdateFiller.Data, 0, (int)removedUpdateFiller.Size); //remove update partition by adding a 32k placeholder. Avoid having a non update partition at 0x50000 nkitDiscInfo.BytesPreservationDiscPadding += removedUpdateFiller.Size; nkitDiscInfo.BytesGaps += fs.Size; dstPos += removedUpdateFiller.Size; crc.Snapshot(string.Format("{0}Replacement Filler", string.IsNullOrEmpty(lastPartitionId) ? "" : (" " + SourceFiles.CleanseFileName(lastPartitionId).PadRight(4)))); if (extractingUpdate) { int storeType; if ((storeType = disc.WriteRecoveryPartitionFiller(updateCrcStream, junk, fs.DiscOffset, true, true, fs, updateTarget, updateTmpFileName, ref updateFileName, updateCrc, true)) != 0) { _log.LogDetail(string.Format("{0}Update recovery partition stored: {1}", storeType == 2 ? "Other " : "", updateFileName)); } extractingUpdate = false; } } else { if (fs.Size != 0) { using (StreamCircularBuffer filler = new StreamCircularBuffer(fs.Size, null, null, output => { foreach (WiiFillerSectionItem item in ((WiiFillerSection)s).Sections) { output.Write(item.Data, 0, (int)item.Size); } })) { Gap gap = new Gap(fs.Size, false); long srcPos = fs.DiscOffset; long gapLen = gap.Encode(filler, ref srcPos, lastPartitionType == PartitionType.Update ? fs.Size : 0x1cL, fs.Size, junk, scrub, target, _log); nkitDiscInfo.BytesPreservationData += gapLen; nkitDiscInfo.BytesGaps += fs.Size; dstPos += gapLen; if (lastPart != null) { lastPart.NewPartitionDataLength += gapLen; } } } //pad partition to 32k int gapLen2 = (int)(dstPos % 0x8000 == 0 ? 0 : 0x8000 - (dstPos % 0x8000)); ByteStream.Zeros.Copy(target, gapLen2); nkitDiscInfo.BytesPreservationDiscPadding += gapLen2; dstPos += gapLen2; if (lastPart != null) { lastPart.NewPartitionDataLength += gapLen2; } if (lastPart != null) { lastPart.Header.WriteUInt32B(0x2bc, (uint)(lastPart.NewPartitionDataLength / 4)); //updates the array in the crc data hdr.Partitions.First(a => a.DiscOffset == lastPart.DiscOffset).DiscOffset = (lastPart.NewDiscOffset - 0x20000); } crc.Snapshot(string.Format("{0}{1}Files+Filler", lastPartitionId ?? "", string.IsNullOrEmpty(lastPartitionId) ? "" : " ")); } } } NkitFormat.LogNkitInfo(nkitDiscInfo, _log, hdr.Id, true); foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null)) { ci.PatchCrc = Crc.Compute(ci.PatchData); } NCrc readerCrcs; uint validationCrc; pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches. if (updateRemoved && crc.Crcs.Length > 2) //freeloader wii only has update partition { hdr.RemoveUpdatePartition(crc.Crcs[2].Offset); } hdr.UpdateOffsets(); hdr.WriteString(0x200, 8, "NKIT v01"); //header and version hdr.Write8(0x60, 1); hdr.Write8(0x61, 1); hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc hdr.WriteUInt32B(0x210, (uint)(imageSize / 4L)); //ctx.ImageLength hdr.WriteUInt32B(0x218, updateCrc.Value); //Update crc - Only if removed crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); crc.Crcs[0].PatchData = hdr.Data; hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); //update with magic applied pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "NKit Written"); } catch (Exception ex) { throw pc.SetWriterException(ex, "NkitWriterWii.Write - Convert"); } }
private long partitionWrite(Stream inStream, NCrc crc, Stream target, WiiPartitionSection pHdr, Context ctx, Coordinator pc, NkitInfo imageInfo, ScrubManager scrub, WiiHashStore hashes, long fstFileAlignment) { #if DEHASH return(pHdr.NewPartitionDataLength = inStream.Copy(target, pHdr.PartitionDataLength, null)); #elif DECRYPT inStream.Copy(target, pHdr.PartitionLength, null); return(pHdr.NewPartitionDataLength = pHdr.PartitionDataLength); #endif MemorySection hdr = MemorySection.Read(inStream, 0x440); //ProgressResult result = ctx.Result; long mlt = 4L; //GC 1L long logicalSize = pHdr.PartitionDataLength; //GC: ctx.ImageLength; long physicalSize = pHdr.PartitionDataLength; //GC: result.ImageInfo.IsoSize; string junkId = hdr.ReadString(0, 4); //bool write = true; List <string> addedFiles = new List <string>(); long srcPos; long dstPos = 0; JunkStream js = new JunkStream(junkId, hdr.Read8(6), logicalSize); try { if (junkId == "\0\0\0\0") { srcPos = hdr.Size; imageInfo.BytesData = srcPos; _log?.LogDetail("Null Partition ID, preserving partition as raw"); ConvertFile cf = new ConvertFile(logicalSize - hdr.Size, true) //Size isn't important for writing //result.ImageInfo.IsoSize { FstFile = new FstFile(null) { DataOffset = hdr.Size, Offset = hdr.Size, Length = 0 }, }; long nullsPos = 0; target.Write(hdr.Data, 0, (int)hdr.Size); //0x400 target.Write(pHdr.Header.Data, 0x2bc, 4); //copy the original partition length dstPos += 0x440 + 4 + NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log); } else { hdr.WriteString(0x200, 8, "NKIT v01"); hdr.WriteUInt32B(0x210, (uint)(pHdr.PartitionLength / mlt)); MemorySection fst; List <JunkDiff> junkDiffs = new List <JunkDiff>(); long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt; //############################################################################ //# READ DISC START target.Write(hdr.Data, 0, (int)hdr.Size); inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size); //read fst with 4 byte boundary fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4)))); crc.Snapshot(junkId + " PrtHdr"); target.Write(fst.Data, 0, (int)fst.Size); crc.Snapshot(junkId + " Fst"); target.Write(hashes.FlagsToByteArray(), 0, (int)hashes.FlagsLength); crc.Snapshot(junkId + " HashFlags"); srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size; long nullsPos = srcPos + 0x1c; dstPos = srcPos + hashes.FlagsLength; //create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample) string error; List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, physicalSize, hdr, fst, false, fstFileAlignment, out error); imageInfo.BytesData = srcPos; if (conFiles == null) { if (error != null) { _log?.LogDetail(error); } ConvertFile cf = new ConvertFile(pHdr.PartitionDataLength - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize { FstFile = new FstFile(null) { DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size }, }; dstPos += NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log); } else { //############################################################################ //# WRITE THE FILESYSTEM List <ConvertFile> missing; NkitFormat.NkitWriteFileSystem(ctx, imageInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out missing, scrub, pHdr.PartitionDataLength, _log); dstPos += hashes.HashesToStream(target); if (missing.Count != 0) { _log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s")); foreach (ConvertFile cf in missing) { _log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length)); } } crc.Crcs[crc.Crcs.Length - 1].PatchData = hashes.FlagsToByteArray(); crc.Crcs[crc.Crcs.Length - 1].PatchCrc = Crc.Compute(crc.Crcs[crc.Crcs.Length - 2].PatchData); crc.Crcs[crc.Crcs.Length - 2].PatchData = fst.Data; crc.Crcs[crc.Crcs.Length - 2].PatchCrc = Crc.Compute(fst.Data); } } } catch (Exception ex) { throw new HandledException(ex, "NkitWriterGc.Write - Convert"); } return(dstPos); }
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc) { try { long mlt = 1L; //for Wii: 4L long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength pc.WriterCheckPoint1WriteReady(out string junkId); //wait until read has written the header and set the length List <string> addedFiles = new List <string>(); NCrc crc = new NCrc(); long srcPos; long dstPos = 0; MemorySection hdr = MemorySection.Read(inStream, 0x440); string id8 = string.Concat(hdr.ReadString(0, 6), hdr.Data[6].ToString("X2"), hdr.Data[7].ToString("X2")); if (junkId == null) { junkId = ctx.Settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == id8)?.JunkId; if (junkId != null) { _log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", junkId, id8.Substring(0, 4))); } } if (junkId == null) { junkId = hdr.ReadString(0, 4); } MemorySection fst; List <JunkDiff> junkDiffs = new List <JunkDiff>(); long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt; long fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == id8)?.Item2 ?? -1; CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //############################################################################ //# READ DISC START target.Write(hdr.Data, 0, (int)hdr.Size); crc.Snapshot("hdr.bin"); inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size); crc.Snapshot("bi2.bin, appldr.bin, main.dol"); //read fst with 4 byte boundary fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4)))); target.Write(fst.Data, 0, (int)fst.Size); crc.Snapshot("fst.bin"); srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size; long nullsPos = srcPos + 0x1c; dstPos = srcPos; //create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample) JunkStream js = new JunkStream(junkId, hdr.Read8(6), NStream.FullSizeGameCube); List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, imageSize, hdr, fst, true, fstFileAlignment, out string error); //Size isn't important for writing //result.ImageInfo.IsoSize NkitInfo nkitInfo = new NkitInfo { BytesData = srcPos, BytesGaps = 0, BytesJunkFiles = 0, BytesPreservationData = 0, BytesPreservationDiscPadding = 0 }; ScrubManager scrub = new ScrubManager(); if (conFiles == null) { if (error != null) { _log?.LogDetail(error); } ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize { FstFile = new FstFile(null) { DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size }, }; NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log); } else { //############################################################################ //# WRITE THE FILESYSTEM NkitFormat.NkitWriteFileSystem(ctx, nkitInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out List <ConvertFile> missing, scrub, imageSize, _log); if (missing.Count != 0) { _log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s")); foreach (ConvertFile cf in missing) { _log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length)); } } } if (dstPos % 0x800 != 0) { long l = 0x800 - (dstPos % 0x800); ByteStream.Zeros.Copy(target, l); dstPos += l; nkitInfo.BytesPreservationDiscPadding += l; } crc.Snapshot("files"); NkitFormat.LogNkitInfo(nkitInfo, _log, hdr.ReadString(0, 4), true); pc.WriterCheckPoint2Complete(out NCrc readerCrcs, out uint validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches. hdr.WriteUInt32B(0x420, (uint)mainDolAddr); hdr.WriteString(0x200, 8, "NKIT v01"); //header and version hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc hdr.WriteUInt32B(0x210, (uint)imageSize); //result.ImageInfo.IsoSize); hdr.WriteString(0x214, 4, hdr.ReadString(0, 4) != junkId ? junkId : "\0\0\0\0"); crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); crc.Crcs[0].PatchData = hdr.Data; crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data); crc.Crcs[2].PatchData = fst.Data; hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, "NKit Written"); } catch (Exception ex) { throw pc.SetWriterException(ex, "NkitWriterGc.Write - Convert"); } }
internal static long ProcessGap(ref long nullsPos, ConvertFile file, ref long srcPos, Stream s, JunkStream junk, bool firstOrLastFile, ScrubManager scrub, Stream output, ILog log) { long nulls = 0; if (file.GapLength != 0) { if (srcPos % 4 != 0) { throw new Exception("Src Position should be on a 4 byte boundary"); } long size = file.GapLength; long start = srcPos; long maxNulls = Math.Max(0, nullsPos - srcPos); //0x1cL //maxNulls = 0x1c; if (size < maxNulls) //need to test this commented if { nulls = size; } else { nulls = size >= 0x40000 && !firstOrLastFile ? 0 : maxNulls; } } //might need to still call if we have a junk file return(file.Gap.Encode(s, ref srcPos, nulls, file.GapLength, junk, scrub, output, log)); }
internal static void NkitWriteFileSystem(Context ctx, NkitInfo imageInfo, long mlt, Stream inStream, ref long srcPos, ref long dstPos, MemorySection hdr, MemorySection fst, ref long mainDolAddr, Stream target, long nullsPos, JunkStream js, List <ConvertFile> conFiles, out List <ConvertFile> missingFiles, ScrubManager scrub, long imageSize, ILog log) { FstFile ff; //########### FILES bool firstFile = true; ConvertFile lastf = conFiles.Last(); ConvertFile prevf = null; missingFiles = new List <ConvertFile>(); foreach (ConvertFile f in conFiles) //read the files and write them out as goodFiles (possible order difference { ff = f.FstFile; if (!firstFile) { if (srcPos == mainDolAddr && mainDolAddr == (hdr.ReadUInt32B(0x420) * mlt)) { mainDolAddr = dstPos; //main/default.dol is moving } //Debug.WriteLine(string.Format(@"{5} : {0} : {1} : {2} : {3}/{4}", ff.DataOffset.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), /*(nextFile.DataOffset - lastEnd).ToString("X8"),*/ ff.Length.ToString("X8"), ff.Path, ff.Name, ff.OffsetInFstFile.ToString("X8"))); //srcPos aligned at 32k and the length is 32k (audio file) ensure new dest is also aligned if (f.Alignment == 0 || (f.Alignment != -1 && dstPos % f.Alignment != 0)) //XGIII audio in race, Zelda collection games for TGC { long pad; if (f.Alignment == 0) //no shrinking { pad = Math.Max(0, ff.DataOffset - dstPos); } else //align { pad = (int)(dstPos % f.Alignment == 0 ? 0 : f.Alignment - (dstPos % f.Alignment)); } imageInfo.FilesAligned += 1; ByteStream.Zeros.Copy(target, pad); imageInfo.BytesPreservationDiscPadding += pad; dstPos += pad; } fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(dstPos / mlt)); //replace copy with junk test - adjust fst length to %4 long l = srcPos; imageInfo.FilesTotal += 1; long written = NkitFormat.CopyFile(ref nullsPos, f, prevf?.FstFile, target, ref srcPos, dstPos, inStream, js, imageSize, out bool missing); if (missing) { missingFiles.Add(f); } dstPos += written; imageInfo.BytesData += srcPos - l; //read if (f.Gap.JunkFile != 0) //encode the gap and write - pad to %4 { imageInfo.BytesJunkFiles += f.Gap.JunkFile + (f.Gap.JunkFile % 4 == 0 ? 0 : (4 - (f.Gap.JunkFile % 4))); fst.WriteUInt32B(ff.OffsetInFstFile + 4, 0); //modify size to be remainder of 4 } } if (f.GapLength != 0 || f.Gap.JunkFile != 0) { long l = NkitFormat.ProcessGap(ref nullsPos, f, ref srcPos, inStream, js, firstFile || f == lastf, scrub, target, log); imageInfo.BytesGaps += f.GapLength; imageInfo.BytesPreservationData += l; dstPos += l; } firstFile = false; prevf = f; } }
private long writeGap(ref long fileLength, LongRef gapLength, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target, JunkStream junk, bool firstOrLastFile, ScrubManager scrub) { if (gapLength.Value == 0) { if (fileLength == 0) { nullsPos = dstPos + 0x1c; } return(0); } long srcLen = gapLength.Value; //fix added for (padding between junk files) - Zumba Fitness (Europe) (En,Fr,De,Es,It) MemorySection ms = MemorySection.Read(inStream, 4); srcPos += 4; long size = ms.ReadUInt32B(0); GapType gt = (GapType)(size & 0b11); size &= 0xFFFFFFFC; if (size == 0xFFFFFFFC) //for wii only. not a thing for GC { srcPos += 4; inStream.Read(ms.Data, 0, 4); size = 0xFFFFFFFCL + (long)ms.ReadUInt32B(0); //cater for files > 0xFFFFFFFF } gapLength.Value = size; scrub.AddGap(fileLength, dstPos, size); //keep track of trailing nulls when restoring scrubbed images long nulls; long junkFileLen = 0; //set nullsPos value if zerobyte file without junk if (gt == GapType.JunkFile) { nullsPos = Math.Min(nullsPos - dstPos, 0); nulls = (size & 0xFC) >> 2; inStream.Read(ms.Data, 0, 4); srcPos += 4; junkFileLen = ms.ReadUInt32B(0); fileLength = junkFileLen; junkFileLen += junkFileLen % 4 == 0 ? 0 : 4 - (junkFileLen % 4); ByteStream.Zeros.Copy(target, nulls); junk.Position = dstPos + nulls; junk.Copy(target, junkFileLen - nulls); dstPos += junkFileLen; if (srcLen <= 8) { return(junkFileLen); } else { //read gap inStream.Read(ms.Data, 0, 4); srcPos += 4; size = ms.ReadUInt32B(0); gt = (GapType)(size & 0b11); size &= 0xFFFFFFFC; gapLength.Value = size; } } else if (fileLength == 0) //last zero byte file was legit { nullsPos = dstPos + 0x1c; } long maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL if (size < maxNulls) //need to test this commented if { nulls = size; } else { nulls = size >= 0x40000 && !firstOrLastFile ? 0 : maxNulls; } nullsPos = dstPos + nulls; //belt and braces if (gt == GapType.AllJunk) { ByteStream.Zeros.Copy(target, nulls); junk.Position = dstPos + nulls; junk.Copy(target, size - nulls); dstPos += size; } else if (gt == GapType.AllScrubbed) { scrub.Scrub(target, dstPos, size, 0); dstPos += size; } else { long prg = size; byte btByte = 0x00; GapBlockType bt = GapBlockType.Junk; //should never be used while (prg > 0) { inStream.Read(ms.Data, 0, 4); srcPos += 4; long bytes; long blk = ms.ReadUInt32B(0); GapBlockType btType = (GapBlockType)(blk >> 30); bool btRepeat = btType == GapBlockType.Repeat; if (!btRepeat) { bt = btType; } long cnt = 0x3FFFFFFF & blk; if (bt == GapBlockType.NonJunk) { bytes = Math.Min(cnt * Gap.BlockSize, prg); inStream.Copy(target, bytes); srcPos += bytes; } else if (bt == GapBlockType.ByteFill) { if (!btRepeat) { btByte = (byte)(0xFF & cnt); //last 8 bits when not repeating are the byte cnt >>= 8; } bytes = Math.Min(cnt * Gap.BlockSize, prg); scrub.Scrub(target, dstPos, bytes, btByte); } else //if (bt == GapBlockType.Junk) { bytes = Math.Min(cnt * Gap.BlockSize, prg); maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL if (prg < maxNulls) { nulls = bytes; } else { nulls = bytes >= 0x40000 && !firstOrLastFile ? 0 : maxNulls; } ByteStream.Zeros.Copy(target, nulls); junk.Position = dstPos + nulls; junk.Copy(target, bytes - nulls); } prg -= bytes; dstPos += bytes; } } return(gapLength.Value + junkFileLen); }
private long writeFiller(ref long srcPos, long dstPos, long nullsPos, Stream inStream, Stream target, JunkStream junk, ScrubManager scrub) { long fileLength = -1; //will be ignored LongRef gapLength = new LongRef() { Value = -1 }; dstPos = writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, dstPos, inStream, target, junk, true, scrub); return(dstPos); }
private long writeGap(ConvertFile file, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target, JunkStream junk, bool firstOrLastFile, ScrubManager scrub) { long fileLength = file.FstFile.Length; LongRef gapLength = new LongRef() { Value = file.GapLength }; dstPos = writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, dstPos, inStream, target, junk, firstOrLastFile, scrub); file.FstFile.Length = fileLength; file.GapLength = gapLength.Value; return(dstPos); }
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc) { try { WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)inStream.DiscHeader; string idVer = hdr.ReadString(0x200, 8); if (idVer != "NKIT v01") { throw new Exception(string.Format("{0} not supported by this version", idVer)); } bool isNkit = idVer.StartsWith("NKIT"); uint nkitCrc = hdr.ReadUInt32B(0x208); long imageSize = hdr.ReadUInt32B(0x210) * 4L; string junkId = hdr.ReadString(0x214, 4); uint updatePartitionCrc = hdr.ReadUInt32B(0x218); MemorySection updateRemovedFiller = null; long discOffset = 0; List <NkitPartitionPatchInfo> patchInfos = new List <NkitPartitionPatchInfo>(); discOffset += hdr.Size; string lastPartitionId = null; PartitionType lastPartitionType = PartitionType.Other; NCrc crc = new NCrc(); long dstPos = 0; long srcPos = hdr.Size; ScrubManager scrubFiller = new ScrubManager(null); bool isRecoverable = false; using (NDisc disc = new NDisc(_log, inStream)) { hdr.WriteUInt32B(0x200, 0); hdr.WriteUInt32B(0x204, 0); hdr.WriteUInt32B(0x208, 0); hdr.WriteUInt32B(0x20C, 0); hdr.WriteUInt32B(0x210, 0); hdr.WriteUInt32B(0x214, 0); hdr.WriteUInt32B(0x218, 0); hdr.Write8(0x60, 0); hdr.Write8(0x61, 0); CryptoStream crcStream = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //wrap to calculate crc crcStream.Write(hdr.Data, 0, hdr.Data.Length); //write the header pc.ReaderCheckPoint1PreWrite(null, nkitCrc); //size that will be output from this read dstPos += hdr.Size; crc.Snapshot("Disc Header"); foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted { if (updatePartitionCrc != 0 && srcPos == hdr.Size) //write update partition out { updateRemovedFiller = MemorySection.Read(inStream, hdr.Partitions.First().DiscOffset - srcPos); srcPos += updateRemovedFiller.Size; WiiPartitionInfo firstPart = WiiDiscHeaderSection.CreatePartitionInfos(updateRemovedFiller, 0)?.FirstOrDefault(a => a.Type != PartitionType.Update); string updateFileName = RecoveryData.GetUpdatePartition(ctx.Settings, updatePartitionCrc); if (updateFileName != null) { using (FileStream pf = File.OpenRead(updateFileName)) { pf.Copy(crcStream, pf.Length); dstPos += pf.Length; } } else { string msg = string.Format("!! Update partition *_{0} missing - Adding filler. It may be Recoverable", updatePartitionCrc.ToString("X8")); isRecoverable = true; _log?.LogDetail(msg); //throw pc.SetReaderException(new HandledException("Failed to convert: " + msg)); } ByteStream.Zeros.Copy(crcStream, firstPart.DiscOffset - dstPos); //fill full gap dstPos += firstPart.DiscOffset - dstPos; } NkitPartitionPatchInfo patchInfo = new NkitPartitionPatchInfo() { HashGroups = new Dictionary <long, MemorySection>() }; patchInfos.Add(patchInfo); if (part.DiscOffset > srcPos) { dstPos += writeFiller(ref srcPos, dstPos, dstPos + 0x1cL, inStream, crcStream, new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize), scrubFiller); inStream.Copy(ByteStream.Zeros, part.DiscOffset - srcPos); //padded to 0x8000 srcPos += part.DiscOffset - srcPos; } part.DiscOffset = dstPos; //restore the original position patchInfo.DiscOffset = dstPos; patchInfo.PartitionHeader = MemorySection.Read(inStream, 0x20000); srcPos += patchInfo.PartitionHeader.Size; long size = patchInfo.PartitionHeader.ReadUInt32B(0x2bc) * 4L; LongRef origSize = new LongRef() { Value = 0 }; WiiPartitionGroupSection wp = null; WiiPartitionHeaderSection wh = new WiiPartitionHeaderSection(hdr, null, part.DiscOffset, patchInfo.PartitionHeader.Data, patchInfo.PartitionHeader.Data.Length); MemorySection ph = new MemorySection(new byte[0x8000 * 64]); long remaining = long.MaxValue;//set after first block read int groupIndex = 0; WiiHashStore hashes = new WiiHashStore(); patchInfo.ScrubManager = wh.ScrubManager; bool patchBlock = false; StreamCircularBuffer prtStream = null; try { using (prtStream = new StreamCircularBuffer(0, null, null, output => srcPos += partitionStreamWrite(origSize, inStream, output, size, ctx.Dats, patchInfo, hashes, pc))) { int gs = 0; int ge = 0; int i = 0; MemoryStream patchBlocks = null; while (remaining > 0) { int blocks = (int)Math.Min(64L, remaining / 0x7c00); for (int b = 0; b < blocks; b++) { prtStream.Read(ph.Data, (b * 0x8000) + 0x400, 0x7c00); //load aligned with no hashes if (remaining == long.MaxValue) //first loop { remaining = origSize.Value; if (ph.ReadString(0x400 + 0, 4) == "\0\0\0\0") { gs = -1; ge = -1; blocks = (int)Math.Min(64L, remaining / 0x7c00); lastPartitionId = ph.ReadString(0x400 + 0, 4); patchInfo.PartitionHeader.WriteUInt32B(0x2bc, (uint)(NStream.DataToHashedLen(origSize.Value) / 4)); //restore real size crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length); dstPos += patchInfo.PartitionHeader.Size; } else { gs = (int)((ph.ReadUInt32B(0x400 + 0x424) * 4L) / (0x7c00L * 64)); ge = (int)(((ph.ReadUInt32B(0x400 + 0x424) * 4L) + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) / (0x7c00L * 64)); if ((int)((part.DiscOffset + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) % (0x7c00L * 64)) == 0) { ge--; //don't load the next group if the data will end on the last byte } blocks = (int)Math.Min(64L, remaining / 0x7c00); lastPartitionId = ph.ReadString(0x400 + 0, 4); patchInfo.PartitionHeader.WriteUInt32B(0x2bc, ph.ReadUInt32B(0x400 + 0x210)); //restore real size crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length); dstPos += patchInfo.PartitionHeader.Size; ph.WriteUInt32B(0x400 + 0x200, 0); ph.WriteUInt32B(0x400 + 0x204, 0); ph.WriteUInt32B(0x400 + 0x208, 0); ph.WriteUInt32B(0x400 + 0x20C, 0); ph.WriteUInt32B(0x400 + 0x210, 0); ph.WriteUInt32B(0x400 + 0x214, 0); ph.WriteUInt32B(0x400 + 0x218, 0); } wp = new WiiPartitionGroupSection(hdr, wh, ph.Data, part.DiscOffset, blocks * 0x8000, false); wh.Initialise(wp, origSize.Value); } } if (blocks < 64) { Array.Clear(ph.Data, blocks * 0x8000, ph.Data.Length - (blocks * 0x8000)); //clear remaining blocks } wp.Populate(groupIndex, ph.Data, dstPos, blocks * 0x8000); int scrubbed = 0; for (int bi = 0; bi < blocks; bi++) { wp.MarkBlockDirty(bi); byte byt; if (patchInfo.ScrubManager.IsBlockScrubbedScanMode(wp.Offset + (bi * 0x8000), out byt)) { wp.SetScrubbed(bi, byt); scrubbed++; } } bool isFstBlocks = i >= gs && i <= ge; bool reqHashes = hashes.IsPreserved(wp.Offset); //test with 0 partition based offset repairBlocks(wp, scrubbed, blocks, false, isFstBlocks); //only test if the hashes aren't preserved (only preserved for scrubbed/customs) if (reqHashes) //store with disc based offset { patchInfo.HashGroups.Add(wp.Offset + part.DiscOffset + patchInfo.PartitionHeader.Size, new MemorySection((byte[])wp.Decrypted.Clone())); //fetch the stored hashed that couldn't be recreated } groupIndex++; bool inFstArea = i >= gs && i <= ge; if (!patchBlock && (gs == i || reqHashes)) { patchBlocks = new MemoryStream(); crc.Snapshot(lastPartitionId + " Data"); patchBlock = true; } else if (patchBlock && !inFstArea && !reqHashes) { crc.Snapshot(lastPartitionId + " Patch"); crc.Crcs.Last().PatchData = patchBlocks.ToArray(); patchBlocks.Dispose(); patchBlock = false; } #if DECRYPT outStream.Write(wp.Decrypted, 0, blocks * 0x8000); if (i >= gs && i <= ge) { fstBlocks.Write(wp.Decrypted, 0, blocks * 0x8000); } #else crcStream.Write(wp.Encrypted, 0, blocks * 0x8000); if (patchBlock) { patchBlocks.Write(wp.Encrypted, 0, blocks * 0x8000); } #endif remaining -= (blocks * 0x7c00); dstPos += (blocks * 0x8000); i++; } if (patchBlock) { crc.Snapshot(lastPartitionId + " Patch"); crc.Crcs.Last().PatchData = patchBlocks.ToArray(); patchBlocks.Dispose(); } } if (origSize.Value != prtStream.WriterPosition) { throw pc.SetReaderException(new HandledException("Partition read did not write the full amount to the circular buffer before completing")); } } catch (Exception ex) { if (prtStream?.WriterException != null) { throw pc.SetReaderException(prtStream.WriterException, "Failed reading filesystem"); } throw pc.SetReaderException(ex, "Failed converting the filesystem");; //writer exception } srcPos += hashes.ReadPatchData(part.DiscOffset + patchInfo.PartitionHeader.Size, patchInfo.HashGroups, inStream); //read hash patches lastPartitionType = part.Type; } if (srcPos < inStream.Length) { JunkStream partJunk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize); dstPos += writeFiller(ref srcPos, dstPos, lastPartitionType == PartitionType.Update ? imageSize : dstPos + 0x1cL, inStream, crcStream, partJunk, scrubFiller); } } crc.Snapshot("End"); if (updatePartitionCrc != 0) { hdr.Write((int)WiiDiscHeaderSection.PartitionTableOffset, updateRemovedFiller.Data, 0, (int)WiiDiscHeaderSection.PartitionTableLength); //restore the exact partition table if update was removed } else { hdr.UpdateOffsets(); //just update the table with the new offsets } crc.Crcs[0].PatchData = hdr.Data; foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null)) { NkitPartitionPatchInfo patchInfo = patchInfos.FirstOrDefault(a => ci.Offset >= a.DiscOffset + a.PartitionHeader.Size && ci.Offset < a.DiscOffset + a.PartitionHeader.Size + a.Size); if (patchInfo != null) { patchGroups(patchInfo, hdr, ci.Offset, ci.PatchData); } ci.PatchCrc = Crc.Compute(ci.PatchData); } if (imageSize != dstPos) { throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString())); } pc.ReaderCheckPoint2Complete(crc, isRecoverable, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid"); pc.ReaderCheckPoint3Complete(); } catch (Exception ex) { throw pc.SetReaderException(ex, "NkitReaderWii.Read - Read and convert"); } }
internal WiiPartitionHeaderSection(WiiDiscHeaderSection header, NStream stream, long discOffset, byte[] data, long size) : base(stream, discOffset, data, size) { _hdr = header; _fileSystem = null; Aes = Aes.Create(); _sha1 = SHA1.Create(); Aes.Padding = PaddingMode.None; _dataOffset = this.ReadUInt32B(0x2b8) * 4L; PartitionSize = this.ReadUInt32B(0x2bc) * 4L; PartitionDataSize = NStream.HashedLenToData(PartitionSize); int h3Offset = (int)this.ReadUInt32B(0x2b4) * 4; int tmdOffset = (int)(this.ReadUInt32B(0x2a8) * 4); if (h3Offset != 0) { H3Table = this.Read(h3Offset, 0x18000); } if (tmdOffset != 0) { ContentSha1 = this.Read(tmdOffset + 0x1e4 + 0x10, 20); } // Determine the common key to use. string issuer = Encoding.ASCII.GetString(this.Read(0x140, 64)).TrimEnd('\0'); IsRvt = issuer == "Root-CA00000002-XS00000006"; //Use the RVT-R key. IsKorean = !IsRvt && this.Read8(0x1f1) == 1; //Use the Korean Key IsRvtR = !(IsRvtH = IsRvt && PartitionSize == 0); if (IsRvtH) { return; //notsupported } int i = IsRvt ? 0 : IsKorean ? 1 : 2; byte[] lame = Convert.FromBase64String(_lame); byte[] l = new byte[lame.Length / 3]; for (int j = 0; j < l.Length; i += 3) { l[j++] = lame[i]; } Aes.Key = l; byte[] titleKey = this.Read(0x1bf, 16); byte[] iv = this.Read(0x1dc, 16); Array.Clear(iv, 8, 8); Aes.IV = iv; using (ICryptoTransform cryptor = Aes.CreateDecryptor()) cryptor.TransformBlock(titleKey, 0, 16, titleKey, 0); this.Key = titleKey; Aes.Key = this.Key; //decrypt scrubbed values. This is to allow the comparison of scrubbed partition data in the decrypted layer DecryptedScrubbed00 = new byte[16]; DecryptedScrubbedFF = new byte[] { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; Aes.IV = (byte[])DecryptedScrubbedFF.Clone(); //if properly scrubbed then the KEY is FFs too using (ICryptoTransform cryptor = Aes.CreateDecryptor()) cryptor.TransformBlock(DecryptedScrubbedFF, 0, 16, DecryptedScrubbedFF, 0); Aes.IV = new byte[16]; using (ICryptoTransform cryptor = Aes.CreateDecryptor()) cryptor.TransformBlock(DecryptedScrubbed00, 0, 16, DecryptedScrubbed00, 0); _scrubManager = new ScrubManager(this); }