Example #1
0
        public long Encode(Stream s, ref long srcPos, long nulls, long gapLength, JunkStream junk, ScrubManager scrub, Stream output, ILog log)
        {
            int  read    = 0;
            long written = 0;

            _headerWritten = false;
            junk.Position  = srcPos;

            if (gapLength != 0)
            {
                int[]  results    = new int[0x400];
                byte[] scrubBytes = new byte[results.Length];
                byte[] buff       = new byte[Math.Min(gapLength, Gap.BlockSize * results.Length)];
                byte[] jbuff      = new byte[buff.Length]; //cache some junk up front so we can thread the block comparisons
                long   start      = srcPos;

                do
                {
                    int req = (int)Math.Min(buff.Length, gapLength - this.Length);
                    Task.WaitAll(
                        Task.Run(() => read = s.Read(buff, 0, req)),
                        Task.Run(() => junk.Read(jbuff, 0, req))
                        );

                    int  blocks = (int)(read / Gap.BlockSize) + (int)Math.Min(1, read % Gap.BlockSize);
                    long pos    = srcPos;
                    Parallel.For(0, blocks, bb =>
                    {
                        byte?scrubByte;
                        if ((results[bb] = blockCompare(nulls, jbuff, junk.JunkLength, scrub, buff, (int)Gap.BlockSize * bb, (int)Math.Min(Gap.BlockSize * (bb + 1), read), pos, start == pos, out scrubByte)) == 1)
                        {
                            scrubBytes[bb] = scrubByte.Value;
                        }
                    });

                    for (int bb = 0; bb < blocks; bb++)
                    {
                        if (results[bb] == 0)
                        {
                            this.Set(); //junk
                        }
                        else if (results[bb] == 1)
                        {
                            this.Set(scrubBytes[bb]); //scrubbed
                            //log?.LogDebug(string.Format(">>> Scrubbed Written: {0} : {1}", srcPos.ToString("X8"), bb.ToString()));
                        }
                        else
                        {
                            //log?.LogDebug(string.Format(">>> NonJunk Written: {0} : {1}", srcPos.ToString("X8"), bb.ToString()));
                            written += this.Set(buff, (int)Gap.BlockSize * bb, (int)Math.Min(Gap.BlockSize * (bb + 1), read), output); //preserve (also data that's part scrubbed part junk in a 256 byte block)
                        }
                    }
                    srcPos += read;
                }while (this.Length < gapLength && read != 0); //sum of processed gap blocks < exact gap
            }

            written += write(output, !_headerWritten);
            return(written);
        }
Example #2
0
 internal WiiFillerSectionItem(NStream stream, long discOffset, byte[] data, long size, bool useBuff, JunkStream junk) : base(stream, discOffset, data, size)
 {
     _useBuff = useBuff;
     _junk    = junk;
     if (_junk != null)
     {
         _junk.Position = discOffset;
         _junkData      = new byte[this.Data.Length];
         _junk.Read(_junkData, 0, (int)base.Size);
         Array.Clear(_junkData, 0, 28);
         base.Data = _useBuff ? data : _junkData;
     }
 }
Example #3
0
        public bool Unscrub(List <JunkRedumpPatch> junkPatches)
        {
            bool changed = false;
            //bool performCheck = true; //start on the assumption last was valid as it would be hassle to work out
            List <FstFile> nulls = new List <FstFile>();

            bool good = _data.IsValid(false);                                                    //forces decrypt and hash cache build and test - does not test data in blocks matchs H0 table

            Parallel.For(0, _data.UsedBlocks, bi => _unscrubValid[bi] = _data.BlockIsValid(bi)); //test data matches H0 table

            for (int bi = 0; bi < _data.UsedBlocks; bi++)
            {
                if (!_unscrubValid[bi])
                {
                    if (_junk == null)
                    {
                        _junk = new JunkStream(_partHdr.Id, _partHdr.DiscNo, _partHdr.PartitionDataSize);
                    }

                    _junk.Position = NStream.OffsetToData(this.Offset + _data.BlockDataOffset(bi), true);
                    _junk.Read(_data.Decrypted, _data.BlockDataOffset(bi), 0x7c00);
                    _data.MarkBlockUnscrubbedAndDirty(bi);
                    changed = true;
                }
            }

            if (junkPatches != null && junkPatches.Count != 0)
            {
                foreach (JunkRedumpPatch jp in junkPatches)
                {
                    if (jp.Offset >= this.DiscOffset && jp.Offset < this.DiscOffset + this.Size)
                    {
                        Array.Copy(jp.Data, 0, _data.Decrypted, jp.Offset - this.DiscOffset, jp.Data.Length);
                        _data.MarkBlockDirty((int)((jp.Offset - this.DiscOffset) / 0x8000));  //560de532
                    }
                }
            }

            if (changed)
            {
                good = _data.IsValid(true); //true as changes were made
                if (!good)
                {
                    bool zerod = false;
                    List <Tuple <long, int, FstFile> > h3Nulls = new List <Tuple <long, int, FstFile> >(_partHdr.ScrubManager.H3Nulls);

                    if (h3Nulls.Count != 0)
                    {
                        int dataLen = (int)NStream.HashedLenToData(this.Decrypted.Length);
                        foreach (Tuple <long, int, FstFile> n in h3Nulls)
                        {
                            if (n.Item1 >= this.DataOffset && n.Item1 < this.DataOffset + dataLen)
                            {
                                int idx = (int)((n.Item1 - this.DataOffset) / 0x7c00);
                                _data.MarkBlockDirty(idx);
                                int pos = (int)NStream.DataToOffset(n.Item1 - this.DataOffset, true);
                                Array.Clear(_data.Decrypted, pos, n.Item2);
                                zerod = true;
                            }
                        }
                    }
                    if (zerod)
                    {
                        good = _data.IsValid(true);
                    }
                    if (!good)
                    {
                        this.H3Errors++;
                    }
                }
            }

            return(changed);
        }
Example #4
0
        public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
        {
            try
            {
                WiiDiscHeaderSection      hdr   = null;
                WiiPartitionHeaderSection pHdr  = null;
                string        lastPartitionId   = null;
                PartitionType lastPartitionType = PartitionType.Other;
                NCrc          crc                 = new NCrc();
                Crc           updateCrc           = new Crc();
                bool          updateRemoved       = false;
                string        updateTmpFileName   = null;
                string        updateFileName      = null;
                bool          extractingUpdate    = false;
                CryptoStream  updateCrcStream     = null;
                NStream       updateTarget        = null;
                CryptoStream  target              = null;
                MemorySection removedUpdateFiller = null;
                int           preservedHashCount  = 0;

                NkitInfo nkitDiscInfo     = new NkitInfo();
                long     fstFileAlignment = -1;

                WiiPartitionSection lastPart = null;

                long dstPos = 0;

                long   imageSize = pc.OutputSize;                 //for Wii: pHdr.PartitionDataLength
                string ignoreJunkId;
                pc.WriterCheckPoint1WriteReady(out ignoreJunkId); //wait until read has written the header and set the length

                NDisc disc = new NDisc(_log, inStream);

                foreach (IWiiDiscSection s in disc.EnumerateSections(imageSize)) //no fixing, image should be good //ctx.ImageLength
                {
                    if (s is WiiDiscHeaderSection)
                    {
                        hdr = (WiiDiscHeaderSection)s;
                        hdr.Write8(0x60, 1);
                        hdr.Write8(0x61, 1);

                        fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == hdr.Id8)?.Item2 ?? -1;

                        target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
                        target.Write(hdr.Data, 0, hdr.Data.Length); //write the header
                        nkitDiscInfo.BytesData += hdr.Size;
                        dstPos += hdr.Size;

                        crc.Snapshot("Disc Header");
                    }
                    else if (s is WiiPartitionSection)
                    {
                        WiiPartitionSection ps     = (WiiPartitionSection)s;
                        WiiHashStore        hashes = new WiiHashStore(ps.PartitionDataLength);
                        ScrubManager        scrub  = ps.Header.ScrubManager;

                        NkitInfo nkitPartInfo = new NkitInfo();

                        if (ps.Header.Type == PartitionType.Update && ctx.Settings.NkitUpdatePartitionRemoval && hdr.Partitions.Count() > 1) //only remove update if there's more partitions
                        {
                            updateRemoved    = true;
                            extractingUpdate = true;

                            pHdr             = ps.Header;
                            extractingUpdate = true;
                            updateCrcStream  = new CryptoStream(ByteStream.Zeros, updateCrc, CryptoStreamMode.Write);
                            crc.Initialize();
                            nkitPartInfo.BytesData += disc.WriteRecoveryPartitionData(updateCrcStream, false, ps, 0, out updateTmpFileName, out updateFileName, out updateTarget);
                            _log?.LogDetail(string.Format("Extracted and Removed {0} Recovery Partition: [{1}]", ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));
                            removedUpdateFiller = new MemorySection(new byte[0x8000]);
                            removedUpdateFiller.Write(0, hdr.Read(0x40000, 0x100)); //backup the original partition table in case it's non standard in some way
                        }
                        else
                        {
                            target.Write(ps.Header.Data, 0, (int)ps.Header.Size);
                            nkitDiscInfo.BytesData += ps.Header.Size;

                            crc.Snapshot(ps.Id + " Hdr");
                            crc.Crcs.Last().PatchData = ps.Header.Data;
                            dstPos          += ps.Header.Size;
                            ps.NewDiscOffset = dstPos;

                            //long written = 0;
                            using (StreamCircularBuffer decrypted = new StreamCircularBuffer(ps.PartitionDataLength, null, null, output =>
                            {
                                //read decrypted partition
                                foreach (WiiPartitionGroupSection pg in ps.Sections)
                                {
                                    nkitPartInfo.BytesHashesData += (pg.Size / 0x8000 * 0x400); //size of input hashes

                                    if (pg.PreserveHashes())
                                    {
                                        nkitPartInfo.BytesHashesPreservation += hashes.Preserve(pg.Offset, pg.Decrypted, pg.Size);
                                        if (++preservedHashCount >= 1500) //too many will bomb the patch caching when converting back. Something is wrong
                                        {
                                            throw pc.SetWriterException(new HandledException("Over 1500 hashes preserved, aborting as image is corrupt or poorly scrubbed."));
                                        }
                                    }
#if !DECRYPT
                                    for (int i = 0; i < pg.Size / 0x8000; i++)
                                    {
                                        output.Write(pg.Decrypted, (i * 0x8000) + 0x400, 0x7c00);
                                    }
#else
                                    output.Write(pg.Decrypted, 0, (int)pg.Size);
#endif
                                }
                            }))
                            {
                                long len = partitionWrite(decrypted, crc, target, ps, ctx, pc, nkitPartInfo, scrub, hashes, fstFileAlignment);
                                ps.NewPartitionDataLength = len;
                                dstPos  += len;
                                lastPart = ps;
                            }
                        }
                        NkitFormat.LogNkitInfo(nkitPartInfo, _log, ps.Id, false);

                        lastPartitionId   = ps.Id;
                        lastPartitionType = ps.Header.Type;
                    }
                    else if (s is WiiFillerSection)
                    {
                        WiiFillerSection fs    = (WiiFillerSection)s;
                        ScrubManager     scrub = new ScrubManager(null);
                        JunkStream       junk  = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);

                        if (lastPartitionType == PartitionType.Update && updateRemoved)
                        {
                            //preserve the original partition table and update filename
                            target.Write(removedUpdateFiller.Data, 0, (int)removedUpdateFiller.Size); //remove update partition by adding a 32k placeholder. Avoid having a non update partition at 0x50000
                            nkitDiscInfo.BytesPreservationDiscPadding += removedUpdateFiller.Size;
                            nkitDiscInfo.BytesGaps += fs.Size;
                            dstPos += removedUpdateFiller.Size;

                            crc.Snapshot(string.Format("{0}Replacement Filler", string.IsNullOrEmpty(lastPartitionId) ? "" : (" " + SourceFiles.CleanseFileName(lastPartitionId).PadRight(4))));
                            if (extractingUpdate)
                            {
                                int storeType;
                                if ((storeType = disc.WriteRecoveryPartitionFiller(updateCrcStream, junk, fs.DiscOffset, true, true, fs, updateTarget, updateTmpFileName, ref updateFileName, updateCrc, true)) != 0)
                                {
                                    _log.LogDetail(string.Format("{0}Update recovery partition stored: {1}", storeType == 2 ? "Other " : "", updateFileName));
                                }
                                extractingUpdate = false;
                            }
                        }
                        else
                        {
                            if (fs.Size != 0)
                            {
                                using (StreamCircularBuffer filler = new StreamCircularBuffer(fs.Size, null, null, output =>
                                {
                                    foreach (WiiFillerSectionItem item in ((WiiFillerSection)s).Sections)
                                    {
                                        output.Write(item.Data, 0, (int)item.Size);
                                    }
                                }))
                                {
                                    Gap  gap    = new Gap(fs.Size, false);
                                    long srcPos = fs.DiscOffset;
                                    long gapLen = gap.Encode(filler, ref srcPos, lastPartitionType == PartitionType.Update ? fs.Size : 0x1cL, fs.Size, junk, scrub, target, _log);
                                    nkitDiscInfo.BytesPreservationData += gapLen;
                                    nkitDiscInfo.BytesGaps             += fs.Size;
                                    dstPos += gapLen;
                                    if (lastPart != null)
                                    {
                                        lastPart.NewPartitionDataLength += gapLen;
                                    }
                                }
                            }
                            //pad partition to 32k
                            int gapLen2 = (int)(dstPos % 0x8000 == 0 ? 0 : 0x8000 - (dstPos % 0x8000));
                            ByteStream.Zeros.Copy(target, gapLen2);
                            nkitDiscInfo.BytesPreservationDiscPadding += gapLen2;
                            dstPos += gapLen2;
                            if (lastPart != null)
                            {
                                lastPart.NewPartitionDataLength += gapLen2;
                            }

                            if (lastPart != null)
                            {
                                lastPart.Header.WriteUInt32B(0x2bc, (uint)(lastPart.NewPartitionDataLength / 4)); //updates the array in the crc data
                                hdr.Partitions.First(a => a.DiscOffset == lastPart.DiscOffset).DiscOffset = (lastPart.NewDiscOffset - 0x20000);
                            }
                            crc.Snapshot(string.Format("{0}{1}Files+Filler", lastPartitionId ?? "", string.IsNullOrEmpty(lastPartitionId) ? "" : " "));
                        }
                    }
                }

                NkitFormat.LogNkitInfo(nkitDiscInfo, _log, hdr.Id, true);

                foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
                {
                    ci.PatchCrc = Crc.Compute(ci.PatchData);
                }

                NCrc readerCrcs;
                uint validationCrc;
                pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.

                if (updateRemoved && crc.Crcs.Length > 2)                                          //freeloader wii only has update partition
                {
                    hdr.RemoveUpdatePartition(crc.Crcs[2].Offset);
                }

                hdr.UpdateOffsets();
                hdr.WriteString(0x200, 8, "NKIT v01"); //header and version
                hdr.Write8(0x60, 1);
                hdr.Write8(0x61, 1);
                hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
                hdr.WriteUInt32B(0x210, (uint)(imageSize / 4L));   //ctx.ImageLength
                hdr.WriteUInt32B(0x218, updateCrc.Value);          //Update crc - Only if removed
                crc.Crcs[0].PatchCrc  = Crc.Compute(hdr.Data);
                crc.Crcs[0].PatchData = hdr.Data;
                hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
                crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);                                                               //update with magic applied

                pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "NKit Written");
            }
            catch (Exception ex)
            {
                throw pc.SetWriterException(ex, "NkitWriterWii.Write - Convert");
            }
        }
Example #5
0
        private long partitionWrite(Stream inStream, NCrc crc, Stream target, WiiPartitionSection pHdr, Context ctx, Coordinator pc, NkitInfo imageInfo, ScrubManager scrub, WiiHashStore hashes, long fstFileAlignment)
        {
#if DEHASH
            return(pHdr.NewPartitionDataLength = inStream.Copy(target, pHdr.PartitionDataLength, null));
#elif DECRYPT
            inStream.Copy(target, pHdr.PartitionLength, null);
            return(pHdr.NewPartitionDataLength = pHdr.PartitionDataLength);
#endif
            MemorySection hdr = MemorySection.Read(inStream, 0x440);

            //ProgressResult result = ctx.Result;
            long   mlt          = 4L;                       //GC 1L
            long   logicalSize  = pHdr.PartitionDataLength; //GC: ctx.ImageLength;
            long   physicalSize = pHdr.PartitionDataLength; //GC: result.ImageInfo.IsoSize;
            string junkId       = hdr.ReadString(0, 4);
            //bool write = true;

            List <string> addedFiles = new List <string>();

            long srcPos;
            long dstPos = 0;

            JunkStream js = new JunkStream(junkId, hdr.Read8(6), logicalSize);

            try
            {
                if (junkId == "\0\0\0\0")
                {
                    srcPos = hdr.Size;
                    imageInfo.BytesData = srcPos;
                    _log?.LogDetail("Null Partition ID, preserving partition as raw");
                    ConvertFile cf = new ConvertFile(logicalSize - hdr.Size, true) //Size isn't important for writing //result.ImageInfo.IsoSize
                    {
                        FstFile = new FstFile(null)
                        {
                            DataOffset = hdr.Size, Offset = hdr.Size, Length = 0
                        },
                    };
                    long nullsPos = 0;
                    target.Write(hdr.Data, 0, (int)hdr.Size); //0x400
                    target.Write(pHdr.Header.Data, 0x2bc, 4); //copy the original partition length
                    dstPos += 0x440 + 4 + NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
                }
                else
                {
                    hdr.WriteString(0x200, 8, "NKIT v01");
                    hdr.WriteUInt32B(0x210, (uint)(pHdr.PartitionLength / mlt));

                    MemorySection   fst;
                    List <JunkDiff> junkDiffs   = new List <JunkDiff>();
                    long            mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;

                    //############################################################################
                    //# READ DISC START
                    target.Write(hdr.Data, 0, (int)hdr.Size);

                    inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);

                    //read fst with 4 byte boundary
                    fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
                    crc.Snapshot(junkId + " PrtHdr");
                    target.Write(fst.Data, 0, (int)fst.Size);
                    crc.Snapshot(junkId + " Fst");
                    target.Write(hashes.FlagsToByteArray(), 0, (int)hashes.FlagsLength);
                    crc.Snapshot(junkId + " HashFlags");

                    srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;

                    long nullsPos = srcPos + 0x1c;
                    dstPos = srcPos + hashes.FlagsLength;

                    //create as late as possible in case id is swaped  - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)

                    string             error;
                    List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, physicalSize, hdr, fst, false, fstFileAlignment, out error);

                    imageInfo.BytesData = srcPos;

                    if (conFiles == null)
                    {
                        if (error != null)
                        {
                            _log?.LogDetail(error);
                        }
                        ConvertFile cf = new ConvertFile(pHdr.PartitionDataLength - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
                        {
                            FstFile = new FstFile(null)
                            {
                                DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
                            },
                        };
                        dstPos += NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
                    }
                    else
                    {
                        //############################################################################
                        //# WRITE THE FILESYSTEM
                        List <ConvertFile> missing;
                        NkitFormat.NkitWriteFileSystem(ctx, imageInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out missing, scrub, pHdr.PartitionDataLength, _log);
                        dstPos += hashes.HashesToStream(target);

                        if (missing.Count != 0)
                        {
                            _log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
                            foreach (ConvertFile cf in missing)
                            {
                                _log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
                            }
                        }
                        crc.Crcs[crc.Crcs.Length - 1].PatchData = hashes.FlagsToByteArray();
                        crc.Crcs[crc.Crcs.Length - 1].PatchCrc  = Crc.Compute(crc.Crcs[crc.Crcs.Length - 2].PatchData);
                        crc.Crcs[crc.Crcs.Length - 2].PatchData = fst.Data;
                        crc.Crcs[crc.Crcs.Length - 2].PatchCrc  = Crc.Compute(fst.Data);
                    }
                }
            }
            catch (Exception ex)
            {
                throw new HandledException(ex, "NkitWriterGc.Write - Convert");
            }
            return(dstPos);
        }
Example #6
0
        public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
        {
            try
            {
                long mlt       = 1L;                               //for Wii: 4L
                long imageSize = pc.OutputSize;                    //for Wii: pHdr.PartitionDataLength
                pc.WriterCheckPoint1WriteReady(out string junkId); //wait until read has written the header and set the length

                List <string> addedFiles = new List <string>();

                NCrc crc = new NCrc();
                long srcPos;
                long dstPos = 0;

                MemorySection hdr = MemorySection.Read(inStream, 0x440);
                string        id8 = string.Concat(hdr.ReadString(0, 6), hdr.Data[6].ToString("X2"), hdr.Data[7].ToString("X2"));

                if (junkId == null)
                {
                    junkId = ctx.Settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == id8)?.JunkId;
                    if (junkId != null)
                    {
                        _log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", junkId, id8.Substring(0, 4)));
                    }
                }

                if (junkId == null)
                {
                    junkId = hdr.ReadString(0, 4);
                }

                MemorySection   fst;
                List <JunkDiff> junkDiffs   = new List <JunkDiff>();
                long            mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;

                long fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == id8)?.Item2 ?? -1;

                CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);

                //############################################################################
                //# READ DISC START
                target.Write(hdr.Data, 0, (int)hdr.Size);
                crc.Snapshot("hdr.bin");

                inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
                crc.Snapshot("bi2.bin, appldr.bin, main.dol");

                //read fst with 4 byte boundary
                fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
                target.Write(fst.Data, 0, (int)fst.Size);
                crc.Snapshot("fst.bin");

                srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;

                long nullsPos = srcPos + 0x1c;
                dstPos = srcPos;

                //create as late as possible in case id is swaped  - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
                JunkStream         js       = new JunkStream(junkId, hdr.Read8(6), NStream.FullSizeGameCube);
                List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, imageSize, hdr, fst, true, fstFileAlignment, out string error); //Size isn't important for writing //result.ImageInfo.IsoSize

                NkitInfo nkitInfo = new NkitInfo
                {
                    BytesData                    = srcPos,
                    BytesGaps                    = 0,
                    BytesJunkFiles               = 0,
                    BytesPreservationData        = 0,
                    BytesPreservationDiscPadding = 0
                };

                ScrubManager scrub = new ScrubManager();
                if (conFiles == null)
                {
                    if (error != null)
                    {
                        _log?.LogDetail(error);
                    }

                    ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
                    {
                        FstFile = new FstFile(null)
                        {
                            DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
                        },
                    };
                    NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
                }
                else
                {
                    //############################################################################
                    //# WRITE THE FILESYSTEM
                    NkitFormat.NkitWriteFileSystem(ctx, nkitInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out List <ConvertFile> missing, scrub, imageSize, _log);
                    if (missing.Count != 0)
                    {
                        _log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
                        foreach (ConvertFile cf in missing)
                        {
                            _log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
                        }
                    }
                }

                if (dstPos % 0x800 != 0)
                {
                    long l = 0x800 - (dstPos % 0x800);
                    ByteStream.Zeros.Copy(target, l);
                    dstPos += l;
                    nkitInfo.BytesPreservationDiscPadding += l;
                }
                crc.Snapshot("files");

                NkitFormat.LogNkitInfo(nkitInfo, _log, hdr.ReadString(0, 4), true);

                pc.WriterCheckPoint2Complete(out NCrc readerCrcs, out uint validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.

                hdr.WriteUInt32B(0x420, (uint)mainDolAddr);

                hdr.WriteString(0x200, 8, "NKIT v01");             //header and version
                hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
                hdr.WriteUInt32B(0x210, (uint)imageSize);          //result.ImageInfo.IsoSize);
                hdr.WriteString(0x214, 4, hdr.ReadString(0, 4) != junkId ? junkId : "\0\0\0\0");

                crc.Crcs[0].PatchCrc  = Crc.Compute(hdr.Data);
                crc.Crcs[0].PatchData = hdr.Data;
                crc.Crcs[2].PatchCrc  = Crc.Compute(fst.Data);
                crc.Crcs[2].PatchData = fst.Data;

                hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
                crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);

                pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, "NKit Written");
            }
            catch (Exception ex)
            {
                throw pc.SetWriterException(ex, "NkitWriterGc.Write - Convert");
            }
        }
Example #7
0
        internal static long CopyFile(ref long nullsPos, ConvertFile conFile, FstFile prevFile, Stream dest, ref long srcPos, long dstPos, Stream srcStream, JunkStream junkNStream, long imageSize, out bool missing)
        {
            //long pos = dest.Position;
            missing = false;
            FstFile file = conFile.FstFile;

            long size = file.Length + (file.Length % 4 == 0 ? 0 : (4 - (file.Length % 4)));

            if (srcPos + size > imageSize)
            {
                size = file.Length; //some rare GC demos end at the end of a non aligned file. This fixes it - v1.2 bugfix
            }

            long written = size;

            byte[] f = new byte[Math.Min(0x30, size)];
            srcStream.Read(f, 0, f.Length);                                                         //then read while junk is created

            if (prevFile != null && prevFile.DataOffset == file.DataOffset && prevFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size
            {
                nullsPos = srcPos + 0x1CL;                                                          //will already be aligned
            }

            int nulls      = (int)(nullsPos - srcPos);
            int countNulls = 0;

            if (f.Length > nulls)
            {
                junkNStream.Position = file.DataOffset; //async junk gen
                for (int i = 0; i < f.Length && f[i] == 0; i++)
                {
                    countNulls++;
                }

                if (countNulls < f.Length) //don't test all nulls
                {
                    missing = junkNStream.Compare(f, 0, f.Length, Math.Max(0, countNulls)) == f.Length;
                }
            }

            if (missing) //start of file is junk
            {
                //check the remainder of the file
                MemorySection junkFile = MemorySection.Read(srcStream, size - f.Length);
                missing = junkNStream.Compare(junkFile.Data, 0, (int)junkFile.Size, 0) == junkFile.Size;

                if (missing)
                {
                    written = 0;
                    conFile.Gap.SetJunkFile((uint)conFile.FstFile.Length, countNulls);
                }
                else //not 100% junk, write the file out
                {
                    dest.Write(f, 0, f.Length);
                    dest.Write(junkFile.Data, 0, (int)junkFile.Size);
                }
                junkFile = null;
            }
            else
            {
                dest.Write(f, 0, f.Length);
                srcStream.Copy(dest, size - f.Length); //copy file
            }

            if (!missing) //reset the gap when no junk
            {
                nullsPos = srcPos + size + 0x1c;
                if (nullsPos % 4 != 0)
                {
                    nullsPos += 4 - (nullsPos % 4);
                }
            }

            srcPos += size;
            return(written);
        }
Example #8
0
        internal static long ProcessGap(ref long nullsPos, ConvertFile file, ref long srcPos, Stream s, JunkStream junk, bool firstOrLastFile, ScrubManager scrub, Stream output, ILog log)
        {
            long nulls = 0;

            if (file.GapLength != 0)
            {
                if (srcPos % 4 != 0)
                {
                    throw new Exception("Src Position should be on a 4 byte boundary");
                }

                long size  = file.GapLength;
                long start = srcPos;

                long maxNulls = Math.Max(0, nullsPos - srcPos); //0x1cL
                                                                //maxNulls = 0x1c;
                if (size < maxNulls)                            //need to test this commented if
                {
                    nulls = size;
                }
                else
                {
                    nulls = size >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
                }
            }
            //might need to still call if we have a junk file
            return(file.Gap.Encode(s, ref srcPos, nulls, file.GapLength, junk, scrub, output, log));
        }
Example #9
0
        internal static void NkitWriteFileSystem(Context ctx, NkitInfo imageInfo, long mlt, Stream inStream, ref long srcPos, ref long dstPos, MemorySection hdr, MemorySection fst, ref long mainDolAddr, Stream target, long nullsPos, JunkStream js, List <ConvertFile> conFiles, out List <ConvertFile> missingFiles, ScrubManager scrub, long imageSize, ILog log)
        {
            FstFile ff;
            //########### FILES
            bool        firstFile = true;
            ConvertFile lastf     = conFiles.Last();
            ConvertFile prevf     = null;

            missingFiles = new List <ConvertFile>();

            foreach (ConvertFile f in conFiles) //read the files and write them out as goodFiles (possible order difference
            {
                ff = f.FstFile;

                if (!firstFile)
                {
                    if (srcPos == mainDolAddr && mainDolAddr == (hdr.ReadUInt32B(0x420) * mlt))
                    {
                        mainDolAddr = dstPos; //main/default.dol is moving
                    }

                    //Debug.WriteLine(string.Format(@"{5} : {0} : {1} : {2} : {3}/{4}", ff.DataOffset.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), /*(nextFile.DataOffset - lastEnd).ToString("X8"),*/ ff.Length.ToString("X8"), ff.Path, ff.Name, ff.OffsetInFstFile.ToString("X8")));

                    //srcPos aligned at 32k and the length is 32k (audio file) ensure new dest is also aligned
                    if (f.Alignment == 0 || (f.Alignment != -1 && dstPos % f.Alignment != 0)) //XGIII audio in race, Zelda collection games for TGC
                    {
                        long pad;
                        if (f.Alignment == 0) //no shrinking
                        {
                            pad = Math.Max(0, ff.DataOffset - dstPos);
                        }
                        else //align
                        {
                            pad = (int)(dstPos % f.Alignment == 0 ? 0 : f.Alignment - (dstPos % f.Alignment));
                        }

                        imageInfo.FilesAligned += 1;
                        ByteStream.Zeros.Copy(target, pad);
                        imageInfo.BytesPreservationDiscPadding += pad;
                        dstPos += pad;
                    }

                    fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(dstPos / mlt));

                    //replace copy with junk test - adjust fst length to %4
                    long l = srcPos;

                    imageInfo.FilesTotal += 1;
                    long written = NkitFormat.CopyFile(ref nullsPos, f, prevf?.FstFile, target, ref srcPos, dstPos, inStream, js, imageSize, out bool missing);
                    if (missing)
                    {
                        missingFiles.Add(f);
                    }

                    dstPos += written;
                    imageInfo.BytesData += srcPos - l; //read

                    if (f.Gap.JunkFile != 0)           //encode the gap and write - pad to %4
                    {
                        imageInfo.BytesJunkFiles += f.Gap.JunkFile + (f.Gap.JunkFile % 4 == 0 ? 0 : (4 - (f.Gap.JunkFile % 4)));
                        fst.WriteUInt32B(ff.OffsetInFstFile + 4, 0); //modify size to be remainder of 4
                    }
                }

                if (f.GapLength != 0 || f.Gap.JunkFile != 0)
                {
                    long l = NkitFormat.ProcessGap(ref nullsPos, f, ref srcPos, inStream, js, firstFile || f == lastf, scrub, target, log);

                    imageInfo.BytesGaps             += f.GapLength;
                    imageInfo.BytesPreservationData += l;
                    dstPos += l;
                }

                firstFile = false;
                prevf     = f;
            }
        }
Example #10
0
        internal int WriteRecoveryPartitionFiller(Stream crcStream, JunkStream junk, long pos, bool isUpdate, bool isNkit, WiiFillerSection fs, NStream target, string tmpFileName, ref string fileName, Crc crc, bool logAsDetail)
        {
            long nullBlocks = 0; //check for random junk - only on a handful of launch releases (Rampage, Ant Bully, Grim Adventures, Happy Feet etc)

            junk.Position = pos;
            long leadingNullsPos = pos;

            foreach (WiiFillerSectionItem fi in fs.Sections)
            {
                crcStream.Write(fi.Data, 0, (int)fi.Size);

                for (int i = 0; i < fi.Size; i += 0x8000)
                {
                    int  len   = (int)Math.Min(0x8000L, fi.Size - (long)i);
                    bool match = junk.Compare(fi.Data, i, len, junk.Position == leadingNullsPos ? 0x1c : 0) == len;

                    if (match)
                    {
                        nullBlocks++;
                    }
                    else
                    {
                        if (nullBlocks != 0)
                        {
                            if (pos == leadingNullsPos)
                            {
                                ByteStream.Zeros.Copy(target, 0x1cL);
                                junk.Position = pos + 0x1cL;
                                junk.Copy(target, (nullBlocks * 0x8000L) - 0x1cL);
                            }
                            else
                            {
                                junk.Position = pos;
                                junk.Copy(target, nullBlocks * 0x8000L);
                            }
                            pos += nullBlocks * 0x8000L;
                        }
                        nullBlocks = 0;
                        target.Write(fi.Data, i, len);
                    }
                }
            }
            target.Close();
            crcStream.Close();

            string fName = (fileName += crc.Value.ToString("X8"));
            bool   redump;

            if (isUpdate)
            {
                redump = this.Settings.RedumpUpdateCrcs.Contains(crc.Value);
            }
            else
            {
                redump = this.Settings.RedumpChannels.FirstOrDefault(a => fName.StartsWith(a.Item1)) != null;
            }
            if (isNkit)
            {
                Directory.CreateDirectory(Settings.NkitRecoveryFilesPath);
            }

            return(storeRecoveryFile(redump, tmpFileName, fileName, Settings.RecoveryFilesPath, isNkit ? Settings.NkitRecoveryFilesPath : Settings.OtherRecoveryFilesPath, logAsDetail));
            //rename the file, delete if dupe
        }
Example #11
0
        /// <summary>
        /// Extracts files from partitions, this is not random access. The iso is read in it's entirety
        /// </summary>
        public ExtractResult ExtractRecoveryFilesWii()
        {
            List <ExtractRecoveryResult> result    = new List <ExtractRecoveryResult>();
            List <WiiPartitionInfo>      toExtract = new List <WiiPartitionInfo>();
            WiiDiscHeaderSection         hdr       = null;
            WiiPartitionHeaderSection    pHdr      = null;
            NStream       target            = null;
            bool          extracting        = false;
            int           channel           = 1;
            string        lastPartitionId   = null;
            PartitionType lastPartitionType = PartitionType.Other;

            string fileName    = null;
            string tmpFileName = null;
            int    extracted   = 0;
            Crc    crc         = new Crc();
            Stream crcStream   = null;
            long   imageSize   = this.NStream.RecoverySize; //for Wii: pHdr.PartitionDataLength


            bool isIso = false; //force to always scrub //Path.GetExtension(_name).ToLower() == ".iso";

            foreach (IWiiDiscSection s in this.EnumerateSectionsFix(false, true, false))
            {
                if (s is WiiDiscHeaderSection)
                {
                    hdr = (WiiDiscHeaderSection)s;

                    Log?.Log(string.Format("Processing: {0}", SourceFileName));

                    toExtract = hdr.Partitions.Where(a => a.Type != PartitionType.Data && a.Type != PartitionType.GameData).ToList();
                    if (toExtract.Count() == 0)
                    {
                        Log?.Log(string.Format("    Skipped: No Recovery Partitions to extract - {0}", SourceFileName));
                        break;
                    }
                }
                else if (s is WiiPartitionSection)
                {
                    WiiPartitionSection ps = (WiiPartitionSection)s;

                    if (!toExtract.Any(a => a.DiscOffset == ps.DiscOffset))
                    {
                        Log?.Log(string.Format("    Skipping {0} Partition: [{1}]...", ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));
                        continue;
                    }
                    extracting = true;

                    pHdr       = ps.Header;
                    extracting = true;
                    Log?.Log(string.Format("    {0} of {1} - Extracting {2} Recovery Partition: [{3}]", (extracted + 1).ToString(), toExtract.Count().ToString(), ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));

                    crcStream = new CryptoStream(ByteStream.Zeros, crc, CryptoStreamMode.Write);
                    crc.Initialize();

                    WriteRecoveryPartitionData(crcStream, isIso, ps, channel, out tmpFileName, out fileName, out target);
                    lastPartitionId   = ps.Id;
                    lastPartitionType = ps.Header.Type;
                }
                else if (s is WiiFillerSection)
                {
                    JunkStream       junk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
                    WiiFillerSection fs   = (WiiFillerSection)s;
                    if (extracting)
                    {
                        int storeType;
                        if ((storeType = WriteRecoveryPartitionFiller(crcStream, junk, fs.DiscOffset, pHdr.Type == PartitionType.Update, false, fs, target, tmpFileName, ref fileName, crc, false)) != 0)
                        {
                            result.Add(new ExtractRecoveryResult()
                            {
                                FileName = fileName, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = false
                            });
                        }

                        if (pHdr.Type != PartitionType.Update)
                        {
                            channel++;
                        }
                        extracted++;
                        extracting = false;
                        bool complete = (extracted == toExtract.Count());
                        if (complete)
                        {
                            break;
                        }
                    }
                }
            }
            return(createExtractResult((Region)hdr.ReadUInt32B(0x4e000), result.ToArray()));
        }
Example #12
0
        private long writeGap(ref long fileLength, LongRef gapLength, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target, JunkStream junk, bool firstOrLastFile, ScrubManager scrub)
        {
            if (gapLength.Value == 0)
            {
                if (fileLength == 0)
                {
                    nullsPos = dstPos + 0x1c;
                }
                return(0);
            }
            long          srcLen = gapLength.Value; //fix added for (padding between junk files) - Zumba Fitness (Europe) (En,Fr,De,Es,It)
            MemorySection ms     = MemorySection.Read(inStream, 4);

            srcPos += 4;
            long    size = ms.ReadUInt32B(0);
            GapType gt   = (GapType)(size & 0b11);

            size &= 0xFFFFFFFC;
            if (size == 0xFFFFFFFC) //for wii only. not a thing for GC
            {
                srcPos += 4;
                inStream.Read(ms.Data, 0, 4);
                size = 0xFFFFFFFCL + (long)ms.ReadUInt32B(0); //cater for files > 0xFFFFFFFF
            }
            gapLength.Value = size;

            scrub.AddGap(fileLength, dstPos, size); //keep track of trailing nulls when restoring scrubbed images

            long nulls;
            long junkFileLen = 0;

            //set nullsPos value if zerobyte file without junk
            if (gt == GapType.JunkFile)
            {
                nullsPos = Math.Min(nullsPos - dstPos, 0);
                nulls    = (size & 0xFC) >> 2;
                inStream.Read(ms.Data, 0, 4);
                srcPos      += 4;
                junkFileLen  = ms.ReadUInt32B(0);
                fileLength   = junkFileLen;
                junkFileLen += junkFileLen % 4 == 0 ? 0 : 4 - (junkFileLen % 4);
                ByteStream.Zeros.Copy(target, nulls);
                junk.Position = dstPos + nulls;
                junk.Copy(target, junkFileLen - nulls);
                dstPos += junkFileLen;

                if (srcLen <= 8)
                {
                    return(junkFileLen);
                }
                else
                {
                    //read gap
                    inStream.Read(ms.Data, 0, 4);
                    srcPos         += 4;
                    size            = ms.ReadUInt32B(0);
                    gt              = (GapType)(size & 0b11);
                    size           &= 0xFFFFFFFC;
                    gapLength.Value = size;
                }
            }
            else if (fileLength == 0) //last zero byte file was legit
            {
                nullsPos = dstPos + 0x1c;
            }


            long maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL

            if (size < maxNulls)                            //need to test this commented if
            {
                nulls = size;
            }
            else
            {
                nulls = size >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
            }
            nullsPos = dstPos + nulls; //belt and braces

            if (gt == GapType.AllJunk)
            {
                ByteStream.Zeros.Copy(target, nulls);
                junk.Position = dstPos + nulls;
                junk.Copy(target, size - nulls);
                dstPos += size;
            }
            else if (gt == GapType.AllScrubbed)
            {
                scrub.Scrub(target, dstPos, size, 0);
                dstPos += size;
            }
            else
            {
                long         prg    = size;
                byte         btByte = 0x00;
                GapBlockType bt     = GapBlockType.Junk; //should never be used

                while (prg > 0)
                {
                    inStream.Read(ms.Data, 0, 4);
                    srcPos += 4;
                    long         bytes;
                    long         blk      = ms.ReadUInt32B(0);
                    GapBlockType btType   = (GapBlockType)(blk >> 30);
                    bool         btRepeat = btType == GapBlockType.Repeat;
                    if (!btRepeat)
                    {
                        bt = btType;
                    }

                    long cnt = 0x3FFFFFFF & blk;

                    if (bt == GapBlockType.NonJunk)
                    {
                        bytes = Math.Min(cnt * Gap.BlockSize, prg);

                        inStream.Copy(target, bytes);
                        srcPos += bytes;
                    }
                    else if (bt == GapBlockType.ByteFill)
                    {
                        if (!btRepeat)
                        {
                            btByte = (byte)(0xFF & cnt); //last 8 bits when not repeating are the byte
                            cnt  >>= 8;
                        }

                        bytes = Math.Min(cnt * Gap.BlockSize, prg);
                        scrub.Scrub(target, dstPos, bytes, btByte);
                    }
                    else //if (bt == GapBlockType.Junk)
                    {
                        bytes = Math.Min(cnt * Gap.BlockSize, prg);

                        maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
                        if (prg < maxNulls)
                        {
                            nulls = bytes;
                        }
                        else
                        {
                            nulls = bytes >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
                        }

                        ByteStream.Zeros.Copy(target, nulls);
                        junk.Position = dstPos + nulls;
                        junk.Copy(target, bytes - nulls);
                    }
                    prg    -= bytes;
                    dstPos += bytes;
                }
            }

            return(gapLength.Value + junkFileLen);
        }
Example #13
0
        private long writeFiller(ref long srcPos, long dstPos, long nullsPos, Stream inStream, Stream target, JunkStream junk, ScrubManager scrub)
        {
            long    fileLength = -1; //will be ignored
            LongRef gapLength  = new LongRef()
            {
                Value = -1
            };

            dstPos = writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, dstPos, inStream, target, junk, true, scrub);
            return(dstPos);
        }
Example #14
0
        private long writeGap(ConvertFile file, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target, JunkStream junk, bool firstOrLastFile, ScrubManager scrub)
        {
            long    fileLength = file.FstFile.Length;
            LongRef gapLength  = new LongRef()
            {
                Value = file.GapLength
            };

            dstPos = writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, dstPos, inStream, target, junk, firstOrLastFile, scrub);
            file.FstFile.Length = fileLength;
            file.GapLength      = gapLength.Value;
            return(dstPos);
        }
Example #15
0
        private long partitionStreamWrite(LongRef outSize, Stream inStream, Stream target, long size, DatData settingsData, NkitPartitionPatchInfo patchInfo, WiiHashStore hashes, Coordinator pc)
        {
            DatData data = settingsData;

            List <string> addedFiles = new List <string>();

            DateTime dt = DateTime.Now;

            MemorySection hdr       = MemorySection.Read(inStream, 0x440);
            long          srcPos    = hdr.Size;
            long          outPos    = 0;
            long          imageSize = 0;

            try
            {
                if (hdr.ReadString(0, 4) == "\0\0\0\0")
                {
                    long    nullsPos   = 0;
                    long    fileLength = -1;
                    LongRef gapLength  = new LongRef()
                    {
                        Value = -1
                    };
                    target.Write(hdr.Data, 0, (int)hdr.Size);
                    MemorySection sz = MemorySection.Read(inStream, 4);
                    srcPos       += 4;
                    outPos       += hdr.Size;
                    imageSize     = sz.ReadUInt32B(0) * 4L;
                    outSize.Value = NStream.HashedLenToData(imageSize);
                    JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), outSize.Value); //SET LENGTH FROM HEADER
                    outPos += writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
                }
                else
                {
                    string idVer = hdr.ReadString(0x200, 8);
                    if (idVer != "NKIT v01")
                    {
                        throw new Exception(string.Format("{0} not supported by this version", idVer));
                    }
                    bool isNkit = idVer.StartsWith("NKIT");
                    imageSize     = NStream.HashedLenToData((hdr.ReadUInt32B(0x210) * 4L));
                    outSize.Value = imageSize;
                    string junkId = hdr.ReadString(0x214, 4);

                    JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), imageSize); //SET LENGTH FROM HEADER

                    MemorySection fst;
                    long          mainDolAddr = hdr.ReadUInt32B(0x420);

                    //############################################################################
                    //# READ DISC START

                    MemorySection hdrToFst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x424) * 4L) - hdr.Size);
                    srcPos += hdrToFst.Size;

                    fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) * 4L);
                    long postFstPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
                    srcPos += fst.Size;

                    hashes.WriteFlagsData(imageSize, inStream);
                    srcPos += hashes.FlagsLength;

                    patchInfo.PartitionDataHeader = hdr;
                    patchInfo.Fst = fst;

                    //############################################################################
                    //# WRITE DISC START

                    target.Write(hdr.Data, 0, (int)hdr.Size);
                    target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
                    target.Write(fst.Data, 0, fst.Data.Length);

                    hdrToFst = null; //let this be collected if needed

                    outPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
                    long               nullsPos = outPos + 0x1c;
                    string             error;
                    List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, size, hdr, fst, false, -1, out error);

                    if (conFiles == null)
                    {
                        if (error != null)
                        {
                            _log?.LogDetail(error);
                        }
                        ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //result.ImageInfo.IsoSize
                        {
                            FstFile = new FstFile(null)
                            {
                                DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
                            },
                        };
                        outPos += writeGap(cf, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
                    }
                    else
                    {
                        conFiles[0].GapLength -= hashes.FlagsLength; //fix for a few customs (no gap between the fst and the first file on the source image, but the hash mask makes it look like there is)
                        //########### FILES
                        bool firstFile = true;
                        for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
                        {
                            ConvertFile f  = conFiles[i];
                            FstFile     ff = f.FstFile;

                            if (!firstFile) //fst already written
                            {
                                //Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), outPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (outPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));

                                if (srcPos < ff.DataOffset)                                  //skip any padding (not written for wii currently)
                                {
                                    inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
                                    srcPos += ff.DataOffset - srcPos;
                                }

                                //write file
                                if (ff.DataOffset == mainDolAddr)
                                {
                                    hdr.WriteUInt32B(0x420, (uint)(outPos / 4L));
                                }
                                fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(outPos / 4L));
                                outPos += copyFile(f, ref nullsPos, ref srcPos, outPos, inStream, target);
                            }

                            if (outPos < imageSize)
                            {
                                long gapLen = writeGap(f, ref nullsPos, ref srcPos, outPos, inStream, target, junk, i == 0 || i == conFiles.Count - 1, patchInfo.ScrubManager);
                                outPos += gapLen;
                                if (!firstFile)
                                {
                                    fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)(ff.Length));
                                }
                            }

                            firstFile = false;
                        }
                    }
                }
                return(srcPos);
            }
            catch (Exception ex)
            {
                throw pc.SetReaderException(ex, "NkitReaderWii.Read - partitionRead");
            }
        }
Example #16
0
        public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
        {
            try
            {
                WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)inStream.DiscHeader;
                string idVer             = hdr.ReadString(0x200, 8);
                if (idVer != "NKIT v01")
                {
                    throw new Exception(string.Format("{0} not supported by this version", idVer));
                }
                bool          isNkit              = idVer.StartsWith("NKIT");
                uint          nkitCrc             = hdr.ReadUInt32B(0x208);
                long          imageSize           = hdr.ReadUInt32B(0x210) * 4L;
                string        junkId              = hdr.ReadString(0x214, 4);
                uint          updatePartitionCrc  = hdr.ReadUInt32B(0x218);
                MemorySection updateRemovedFiller = null;

                long discOffset = 0;
                List <NkitPartitionPatchInfo> patchInfos = new List <NkitPartitionPatchInfo>();
                discOffset += hdr.Size;
                string        lastPartitionId   = null;
                PartitionType lastPartitionType = PartitionType.Other;
                NCrc          crc           = new NCrc();
                long          dstPos        = 0;
                long          srcPos        = hdr.Size;
                ScrubManager  scrubFiller   = new ScrubManager(null);
                bool          isRecoverable = false;

                using (NDisc disc = new NDisc(_log, inStream))
                {
                    hdr.WriteUInt32B(0x200, 0);
                    hdr.WriteUInt32B(0x204, 0);
                    hdr.WriteUInt32B(0x208, 0);
                    hdr.WriteUInt32B(0x20C, 0);
                    hdr.WriteUInt32B(0x210, 0);
                    hdr.WriteUInt32B(0x214, 0);
                    hdr.WriteUInt32B(0x218, 0);

                    hdr.Write8(0x60, 0);
                    hdr.Write8(0x61, 0);

                    CryptoStream crcStream = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //wrap to calculate crc
                    crcStream.Write(hdr.Data, 0, hdr.Data.Length);                                     //write the header
                    pc.ReaderCheckPoint1PreWrite(null, nkitCrc);                                       //size that will be output from this read
                    dstPos += hdr.Size;

                    crc.Snapshot("Disc Header");

                    foreach (WiiPartitionInfo part in hdr.Partitions)      //already sorted
                    {
                        if (updatePartitionCrc != 0 && srcPos == hdr.Size) //write update partition out
                        {
                            updateRemovedFiller = MemorySection.Read(inStream, hdr.Partitions.First().DiscOffset - srcPos);
                            srcPos += updateRemovedFiller.Size;
                            WiiPartitionInfo firstPart      = WiiDiscHeaderSection.CreatePartitionInfos(updateRemovedFiller, 0)?.FirstOrDefault(a => a.Type != PartitionType.Update);
                            string           updateFileName = RecoveryData.GetUpdatePartition(ctx.Settings, updatePartitionCrc);

                            if (updateFileName != null)
                            {
                                using (FileStream pf = File.OpenRead(updateFileName))
                                {
                                    pf.Copy(crcStream, pf.Length);
                                    dstPos += pf.Length;
                                }
                            }
                            else
                            {
                                string msg = string.Format("!! Update partition *_{0} missing - Adding filler. It may be Recoverable", updatePartitionCrc.ToString("X8"));
                                isRecoverable = true;
                                _log?.LogDetail(msg);
                                //throw pc.SetReaderException(new HandledException("Failed to convert: " + msg));
                            }
                            ByteStream.Zeros.Copy(crcStream, firstPart.DiscOffset - dstPos); //fill full gap
                            dstPos += firstPart.DiscOffset - dstPos;
                        }

                        NkitPartitionPatchInfo patchInfo = new NkitPartitionPatchInfo()
                        {
                            HashGroups = new Dictionary <long, MemorySection>()
                        };
                        patchInfos.Add(patchInfo);

                        if (part.DiscOffset > srcPos)
                        {
                            dstPos += writeFiller(ref srcPos, dstPos, dstPos + 0x1cL, inStream, crcStream, new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize), scrubFiller);
                            inStream.Copy(ByteStream.Zeros, part.DiscOffset - srcPos); //padded to 0x8000
                            srcPos += part.DiscOffset - srcPos;
                        }

                        part.DiscOffset           = dstPos; //restore the original position
                        patchInfo.DiscOffset      = dstPos;
                        patchInfo.PartitionHeader = MemorySection.Read(inStream, 0x20000);
                        srcPos += patchInfo.PartitionHeader.Size;
                        long    size     = patchInfo.PartitionHeader.ReadUInt32B(0x2bc) * 4L;
                        LongRef origSize = new LongRef()
                        {
                            Value = 0
                        };
                        WiiPartitionGroupSection  wp = null;
                        WiiPartitionHeaderSection wh = new WiiPartitionHeaderSection(hdr, null, part.DiscOffset, patchInfo.PartitionHeader.Data, patchInfo.PartitionHeader.Data.Length);
                        MemorySection             ph = new MemorySection(new byte[0x8000 * 64]);
                        long         remaining       = long.MaxValue;//set after first block read
                        int          groupIndex      = 0;
                        WiiHashStore hashes          = new WiiHashStore();
                        patchInfo.ScrubManager = wh.ScrubManager;
                        bool patchBlock = false;
                        StreamCircularBuffer prtStream = null;

                        try
                        {
                            using (prtStream = new StreamCircularBuffer(0, null, null, output => srcPos += partitionStreamWrite(origSize, inStream, output, size, ctx.Dats, patchInfo, hashes, pc)))
                            {
                                int          gs          = 0;
                                int          ge          = 0;
                                int          i           = 0;
                                MemoryStream patchBlocks = null;

                                while (remaining > 0)
                                {
                                    int blocks = (int)Math.Min(64L, remaining / 0x7c00);
                                    for (int b = 0; b < blocks; b++)
                                    {
                                        prtStream.Read(ph.Data, (b * 0x8000) + 0x400, 0x7c00); //load aligned with no hashes

                                        if (remaining == long.MaxValue)                        //first loop
                                        {
                                            remaining = origSize.Value;

                                            if (ph.ReadString(0x400 + 0, 4) == "\0\0\0\0")
                                            {
                                                gs              = -1;
                                                ge              = -1;
                                                blocks          = (int)Math.Min(64L, remaining / 0x7c00);
                                                lastPartitionId = ph.ReadString(0x400 + 0, 4);
                                                patchInfo.PartitionHeader.WriteUInt32B(0x2bc, (uint)(NStream.DataToHashedLen(origSize.Value) / 4)); //restore real size
                                                crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
                                                dstPos += patchInfo.PartitionHeader.Size;
                                            }
                                            else
                                            {
                                                gs = (int)((ph.ReadUInt32B(0x400 + 0x424) * 4L) / (0x7c00L * 64));
                                                ge = (int)(((ph.ReadUInt32B(0x400 + 0x424) * 4L) + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) / (0x7c00L * 64));
                                                if ((int)((part.DiscOffset + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) % (0x7c00L * 64)) == 0)
                                                {
                                                    ge--; //don't load the next group if the data will end on the last byte
                                                }
                                                blocks          = (int)Math.Min(64L, remaining / 0x7c00);
                                                lastPartitionId = ph.ReadString(0x400 + 0, 4);

                                                patchInfo.PartitionHeader.WriteUInt32B(0x2bc, ph.ReadUInt32B(0x400 + 0x210)); //restore real size
                                                crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
                                                dstPos += patchInfo.PartitionHeader.Size;

                                                ph.WriteUInt32B(0x400 + 0x200, 0);
                                                ph.WriteUInt32B(0x400 + 0x204, 0);
                                                ph.WriteUInt32B(0x400 + 0x208, 0);
                                                ph.WriteUInt32B(0x400 + 0x20C, 0);
                                                ph.WriteUInt32B(0x400 + 0x210, 0);
                                                ph.WriteUInt32B(0x400 + 0x214, 0);
                                                ph.WriteUInt32B(0x400 + 0x218, 0);
                                            }
                                            wp = new WiiPartitionGroupSection(hdr, wh, ph.Data, part.DiscOffset, blocks * 0x8000, false);
                                            wh.Initialise(wp, origSize.Value);
                                        }
                                    }

                                    if (blocks < 64)
                                    {
                                        Array.Clear(ph.Data, blocks * 0x8000, ph.Data.Length - (blocks * 0x8000)); //clear remaining blocks
                                    }
                                    wp.Populate(groupIndex, ph.Data, dstPos, blocks * 0x8000);

                                    int scrubbed = 0;
                                    for (int bi = 0; bi < blocks; bi++)
                                    {
                                        wp.MarkBlockDirty(bi);
                                        byte byt;
                                        if (patchInfo.ScrubManager.IsBlockScrubbedScanMode(wp.Offset + (bi * 0x8000), out byt))
                                        {
                                            wp.SetScrubbed(bi, byt);
                                            scrubbed++;
                                        }
                                    }
                                    bool isFstBlocks = i >= gs && i <= ge;
                                    bool reqHashes   = hashes.IsPreserved(wp.Offset);                                                                                            //test with 0 partition based offset

                                    repairBlocks(wp, scrubbed, blocks, false, isFstBlocks);                                                                                      //only test if the hashes aren't preserved (only preserved for scrubbed/customs)

                                    if (reqHashes)                                                                                                                               //store with disc based offset
                                    {
                                        patchInfo.HashGroups.Add(wp.Offset + part.DiscOffset + patchInfo.PartitionHeader.Size, new MemorySection((byte[])wp.Decrypted.Clone())); //fetch the stored hashed that couldn't be recreated
                                    }
                                    groupIndex++;
                                    bool inFstArea = i >= gs && i <= ge;

                                    if (!patchBlock && (gs == i || reqHashes))
                                    {
                                        patchBlocks = new MemoryStream();
                                        crc.Snapshot(lastPartitionId + " Data");
                                        patchBlock = true;
                                    }
                                    else if (patchBlock && !inFstArea && !reqHashes)
                                    {
                                        crc.Snapshot(lastPartitionId + " Patch");
                                        crc.Crcs.Last().PatchData = patchBlocks.ToArray();
                                        patchBlocks.Dispose();
                                        patchBlock = false;
                                    }
#if DECRYPT
                                    outStream.Write(wp.Decrypted, 0, blocks * 0x8000);
                                    if (i >= gs && i <= ge)
                                    {
                                        fstBlocks.Write(wp.Decrypted, 0, blocks * 0x8000);
                                    }
#else
                                    crcStream.Write(wp.Encrypted, 0, blocks * 0x8000);
                                    if (patchBlock)
                                    {
                                        patchBlocks.Write(wp.Encrypted, 0, blocks * 0x8000);
                                    }
#endif

                                    remaining -= (blocks * 0x7c00);
                                    dstPos    += (blocks * 0x8000);
                                    i++;
                                }
                                if (patchBlock)
                                {
                                    crc.Snapshot(lastPartitionId + " Patch");
                                    crc.Crcs.Last().PatchData = patchBlocks.ToArray();
                                    patchBlocks.Dispose();
                                }
                            }
                            if (origSize.Value != prtStream.WriterPosition)
                            {
                                throw pc.SetReaderException(new HandledException("Partition read did not write the full amount to the circular buffer before completing"));
                            }
                        }
                        catch (Exception ex)
                        {
                            if (prtStream?.WriterException != null)
                            {
                                throw pc.SetReaderException(prtStream.WriterException, "Failed reading filesystem");
                            }
                            throw pc.SetReaderException(ex, "Failed converting the filesystem");;  //writer exception
                        }

                        srcPos += hashes.ReadPatchData(part.DiscOffset + patchInfo.PartitionHeader.Size, patchInfo.HashGroups, inStream);

                        //read hash patches
                        lastPartitionType = part.Type;
                    }

                    if (srcPos < inStream.Length)
                    {
                        JunkStream partJunk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
                        dstPos += writeFiller(ref srcPos, dstPos, lastPartitionType == PartitionType.Update ? imageSize : dstPos + 0x1cL, inStream, crcStream, partJunk, scrubFiller);
                    }
                }

                crc.Snapshot("End");

                if (updatePartitionCrc != 0)
                {
                    hdr.Write((int)WiiDiscHeaderSection.PartitionTableOffset, updateRemovedFiller.Data, 0, (int)WiiDiscHeaderSection.PartitionTableLength); //restore the exact partition table if update was removed
                }
                else
                {
                    hdr.UpdateOffsets(); //just update the table with the new offsets
                }
                crc.Crcs[0].PatchData = hdr.Data;

                foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
                {
                    NkitPartitionPatchInfo patchInfo = patchInfos.FirstOrDefault(a => ci.Offset >= a.DiscOffset + a.PartitionHeader.Size && ci.Offset < a.DiscOffset + a.PartitionHeader.Size + a.Size);
                    if (patchInfo != null)
                    {
                        patchGroups(patchInfo, hdr, ci.Offset, ci.PatchData);
                    }
                    ci.PatchCrc = Crc.Compute(ci.PatchData);
                }

                if (imageSize != dstPos)
                {
                    throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
                }

                pc.ReaderCheckPoint2Complete(crc, isRecoverable, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
                pc.ReaderCheckPoint3Complete();
            }
            catch (Exception ex)
            {
                throw pc.SetReaderException(ex, "NkitReaderWii.Read - Read and convert");
            }
        }