private long copyFile(ConvertFile file, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target) { FstFile ff = file.FstFile; long size = ff.Length; if (size == 0) { return(0); //could be legit or junk } size += size % 4 == 0 ? 0 : 4 - (size % 4); try { inStream.Copy(target, size); } catch (Exception ex) { throw new HandledException(ex, "Copy file '{0}' failed at Data Position {1} ({2} bytes)", ff.Name, ff.DataOffset.ToString("X"), ff.Length.ToString()); } srcPos += size; dstPos += size; nullsPos = dstPos + 0x1CL; return(size); }
public static FileSystem Parse(Stream fstData, long fstOffset, long length, string id, bool isGc) { MemorySection ms = MemorySection.Read(fstData, length); FstFile ff = new FstFile(null) { Name = "fst.bin", DataOffset = fstOffset, Offset = NStream.DataToOffset(fstOffset, !isGc), IsNonFstFile = true, Length = (int)fstData.Length }; return(Parse(ms, ff, id, isGc)); }
internal void ParseFst(byte[] fst) { if (_fileSystem == null && _fst == null) { _fst = fst; } if (this.FileSystem == null) { return; } _fileSystem = this.FileSystem; //force read of filesystem here List <Tuple <long, int, FstFile> > h3 = new List <Tuple <long, int, FstFile> >(); List <FstFile> nonNull = _fileSystem.Files.Where(a => a.Length != 0).ToList(); for (int i = 0; i < nonNull.Count; i++) //already ordered by offset { FstFile fl = nonNull[i]; FstFile fn = null; if (i + 1 < nonNull.Count) { fn = nonNull[i + 1]; } long end = fl.DataOffset + fl.Length; if (end % 0x7c00L == 0 || (int)(end % 0x7c00L) > 0x7c00 - 28) { if (fn == null) { h3.Add(new Tuple <long, int, FstFile>(fl.DataOffset, 28, fl)); } else { int nullCount = 28; if (end % 0x7c00L != 0) { nullCount = (0x7c00 - (int)(end % 0x7c00L)); end += nullCount; nullCount = 28 - nullCount; } long diff = (int)(fn.DataOffset - end); if (diff >= 0x7c00) { h3.Add(new Tuple <long, int, FstFile>(end, nullCount + (nullCount % 2), fl)); //data offset } } } } _scrubManager.H3Nulls.AddRange(h3.Union(_fileSystem.Files.Where(a => a.Length == 0).Select(a => new Tuple <long, int, FstFile>(a.DataOffset, 28, a)))); }
internal static FileSystem Parse(MemorySection ms, FstFile fst, string id, bool isGc) { FstFolder fld = new FstFolder(null); long nFiles = ms.ReadUInt32B(0x8); if (12 * nFiles > ms.Size) { return(null); } if (fst != null) { fld.Files.Add(fst); } recurseFst(ms, fld, 12 * nFiles, 0, id, isGc); return(new FileSystem(fld)); }
private long copyFile(ConvertFile file, ref long nullsPos, ref long srcPos, long dstPos, long imageSize, Stream inStream, Stream target) { FstFile ff = file.FstFile; long size = ff.Length; if (size == 0) { return(0); //could be legit or junk } size += size % 4 == 0 ? 0 : 4 - (size % 4); size = Math.Min(size, imageSize - dstPos); //never overwrite file length (starfox e3 fix) inStream.Copy(target, size); srcPos += size; dstPos += size; nullsPos = dstPos + 0x1CL; return(size); }
private bool areaUsed(FstFile file, long offset, long size) { bool b = (file.DataOffset > offset && file.DataOffset < offset + size); //starts within range if (b) { return(true); } b = (file.DataOffset + file.Length > offset && file.DataOffset + file.Length < offset + size); //ends within range if (b) { return(true); } b = (file.DataOffset <offset && file.DataOffset + file.Length> offset + size); if (b) { return(true); } return(false); }
private long writeFile(ref long nullsPos, Stream dest, long dstPos, ref bool firstFile, Stream srcStream, FstFile lastFile, FstFile file, List <JunkRedumpPatch> patches, NStream nstream, long junkStart, bool isJunkFile) { //Debug.WriteLine(string.Format(@"{0} : {1} : {2} : {3}/{4}", file.DataOffset.ToString("X8"), (file.DataOffset + file.Length).ToString("X8"), /*(nextFile.DataOffset - lastEnd).ToString("X8"),*/ file.Length.ToString("X8"), file.Path, file.Name)); //file found long gap = writeDestGap(nullsPos, dest, dstPos, file.DataOffset - dstPos, firstFile, patches, junkStart, nstream); firstFile = false; bool missing = false; if (file.Length == 0) { // missing = true; } else if (isJunkFile) { missing = true; MemorySection ms = MemorySection.Read(srcStream, 8); long size = ms.ReadUInt32B(0); GapType gt = (GapType)(size & 0b11); size &= 0xFFFFFFFC; //set nullsPos value if zerobyte file without junk if (gt == GapType.JunkFile) { nullsPos = Math.Min(nullsPos - (dstPos + gap), 0); //reset the nulls long nulls = (size & 0xFC) >> 2; long junkFileLen = ms.ReadUInt32B(4); if (junkFileLen != file.Length) { throw new HandledException(string.Format("NKit Junk file restoration length mismatch {0}: {1}", file.DataOffset.ToString("X8"), file.Name)); } ByteStream.Zeros.Copy(dest, nulls); nstream.JunkStream.Position = (dstPos + gap) + nulls; nstream.JunkStream.Copy(dest, junkFileLen - nulls); _log?.LogDetail(string.Format("Generated file content with Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name)); } else { throw new HandledException(string.Format("NKit Junk file restoration bytes invalid {0}: {1}", file.DataOffset.ToString("X8"), file.Name)); } } else { byte[] f = new byte[Math.Min(0x30, file.Length)]; srcStream.Read(f, 0, f.Length); //then read while junk is created if (lastFile.DataOffset == file.DataOffset && lastFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size { nullsPos = dstPos + 0x1CL; //will already be aligned } int nulls = (int)(nullsPos - (dstPos + gap)); nstream.JunkStream.Position = file.DataOffset; //async junk gen int countNulls = 0; for (int i = 0; i < f.Length && f[i] == 0; i++) { countNulls++; } if (f.Length > nulls && countNulls < f.Length) //don't test all nulls { missing = nstream.JunkStream.Compare(f, 0, f.Length, Math.Max(0, nulls)) == f.Length; } if (missing) { _log?.LogDetail(string.Format("File content is Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name)); } dest.Write(f, 0, f.Length); srcStream.Copy(dest, file.Length - f.Length); //copy file } if (!missing) //reset the gap when no junk { nullsPos = dstPos + gap + file.Length + 0x1c; if (nullsPos % 4 != 0) { nullsPos += 4 - (nullsPos % 4); } } return(gap + file.Length); }
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc) { string resultMsg = ""; if (!Settings.ConfigFileFound) { _log?.Log("!! No config file found - This is required to restore and validate images"); } NCrc crc = new NCrc(); Settings settings = ctx.Settings; List <string> addedFiles = new List <string>(); DatData data = ctx.Dats; RecoveryData rec = ctx.Recovery; List <JunkRedumpPatch> patches = rec.JunkPatches; MemorySection hdr = null; try { long junkStart = settings.JunkStartOffsets.FirstOrDefault(a => a.Id8 == inStream.Id8)?.Offset ?? 0; string forceJunkId = settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == inStream.Id8)?.JunkId; if (forceJunkId != null) { _log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", forceJunkId, inStream.Id)); //result.ImageInfo.JunkId = forceJunkId; inStream.ChangeJunk(forceJunkId); } if (junkStart != 0) { _log?.LogDetail(string.Format("Junk forced to start at 0x{0}", junkStart.ToString("X8"))); } hdr = inStream.DiscHeader; FstFileItem goodFst = rec.GcBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8 && !(inStream.Id8 == "GNHE5d0000" && a.Length == 116)); if (goodFst != null) { _log?.LogDetail(string.Format("Recovery: {0}", goodFst.Filename)); } if (goodFst == null && rec.GcNewBinFiles != null) { goodFst = rec.GcNewBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8); } if (goodFst != null) { goodFst.Populate(); } ApploaderFileItem goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc); if (goodAldr == null && rec.GcNewBinFiles != null) { goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcNewBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc); } //is it an action replay (custom hacks) if ((hdr.ReadUInt32B(0x420) == 0 && inStream.Id8 == "GNHE5d0000") || inStream.Id8 == "DTLX010000" || inStream.Id8 == "102E010007") { if (inStream.Id8 == "102E010007") { resultMsg = "Aging Disc detected - Skipping recover"; } else { resultMsg = "Datel Action Replay detected - Skipping recover"; } try { using (CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write)) { target.Write(hdr.Data, 0, (int)hdr.Size); pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read inStream.Copy(target, pc.OutputSize - hdr.Size); } _log?.LogDetail(resultMsg); } catch (Exception ex) { throw new HandledException(ex, resultMsg); } crc.Snapshot("files"); } else { //############################################################################ //# READ DISC START //########### Header (boot.bin) 0 (read by base stream already) long srcPos = hdr.Size; _log?.LogDetail(string.Format("Header Read: {0} - {1}", friendly(inStream.Id), friendly(inStream.Title))); //########### Header Info (bi2.bin) 0x440 MemorySection bi2bin = MemorySection.Read(inStream, 0x2000); srcPos += bi2bin.Size; //########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol MemorySection appldr = MemorySection.Read(inStream, Math.Min(hdr.ReadUInt32B(0x420) == 0 ? uint.MaxValue : hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - srcPos); srcPos += appldr.Size; //########### APP (main.dol) MemorySection maindol; if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424)) { maindol = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - srcPos); } else { maindol = new MemorySection(new byte[0]); } srcPos += maindol.Size; //########### FST (fst.bin) MemorySection srcFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428)); srcPos += hdr.ReadUInt32B(0x428); //############################################################################ //# CORRECT ISSUES List <FstFile> srcFiles = FileSystem.Parse(srcFst, null, inStream.Id, true)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList(); if (srcFiles == null) { throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", hdr.ReadUInt32B(0x424).ToString("X8"))); } uint crcTmp = Crc.Compute(appldr.Data, 0, Math.Min((int)appldr.Size, 0x20 + (int)(appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18)))); //adjust appldr if (goodAldr != null && crcTmp != goodAldr.Crc) { _log?.LogDetail(string.Format("Replacing appldr.bin crc {0} with Recovery appldr.bin {1}", crcTmp.ToString("X8"), goodAldr.Crc.ToString("X8"))); addedFiles.Add(goodAldr.Filename); appldr = new MemorySection(File.ReadAllBytes(goodAldr.Filename)); } //adjust main.dol if (goodFst != null) { if (goodFst.MainDolOffset != hdr.ReadUInt32B(0x420)) { //main.dol is before fst in image and after in goodfst if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424) && goodFst.MainDolOffset > goodFst.FstOffset) { _log?.LogDetail(string.Format("Skipping main.dol at 0x{0} using 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8"))); maindol = new MemorySection(new byte[0]); } else { _log?.LogDetail(string.Format("Moving main.dol address 0x{0} to 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8"))); } hdr.WriteUInt32B(0x420, (uint)goodFst.MainDolOffset); } if (goodFst.Region != (Region)bi2bin.ReadUInt32B(0x18)) { _log?.LogDetail(string.Format("Region Changed to {0} from {1}", goodFst.Region.ToString(), ((Region)bi2bin.ReadUInt32B(0x18)).ToString())); bi2bin.WriteUInt32B(0x18, (uint)goodFst.Region); } if (goodFst.MaxFst != hdr.ReadUInt32B(0x42C)) { _log?.LogDetail(string.Format("Max Fst Size changed to {0} from {1}", goodFst.MaxFst.ToString("X8"), hdr.ReadUInt32B(0x42C).ToString("X8"))); hdr.WriteUInt32B(0x42C, (uint)goodFst.MaxFst); } string newTitle = Encoding.ASCII.GetString(goodFst.Title); if (newTitle != hdr.ReadString(0x20, 0x60 - 0x20)) { _log?.LogDetail(string.Format("Title changed to '{0}' from '{1}'", newTitle.TrimEnd('\0'), hdr.ReadString(0x20, 0x60 - 0x20).TrimEnd('\0'))); hdr.Write(0x20, goodFst.Title); } } MemorySection fst = srcFst; List <FstFile> fstFiles = srcFiles; crcTmp = Crc.Compute(srcFst.Data); if (goodFst != null && crcTmp != goodFst.Crc) { _log?.LogDetail(string.Format("Replacing fst.bin crc {0} with Recovery fst {1}", crcTmp.ToString("X8"), goodFst.Crc.ToString("X8"))); addedFiles.Add(goodFst.Filename); fst = new MemorySection(goodFst.FstData); fstFiles = FileSystem.Parse(fst, null, inStream.Id, true).Files.OrderBy(a => a.Offset).ThenBy(a => a.Length).ToList(); } //adjust fst.bin if (goodFst != null && (goodFst.FstOffset != hdr.ReadUInt32B(0x424) || goodFst.FstData.Length != hdr.ReadUInt32B(0x428))) { _log?.LogDetail(string.Format("Moving / Resizing fst.bin address 0x{0} (Length {1}) to 0x{2} (Length {3})", hdr.ReadUInt32B(0x424).ToString("X8"), hdr.ReadUInt32B(0x428).ToString(), goodFst.FstOffset.ToString("X8"), goodFst.FstData.Length.ToString())); hdr.WriteUInt32B(0x424, (uint)goodFst.FstOffset); hdr.WriteUInt32B(0x428, (uint)goodFst.FstData.Length); } int c; int failCount = 0; int filesMoved = 0; int nkitJunkFiles = 0; foreach (FstFile f in srcFiles) { FstFile[] fnd = fstFiles.Where(a => a.Name == f.Name && a.Path == f.Path).ToArray(); if (fnd.Length == 0) { failCount++; _log?.LogDetail(string.Format("FST Error: No File Found - {0}/{1} (Length {2})", f.Path, f.Name, f.Length.ToString())); } else if (inStream.IsNkit && (c = fnd.Count(a => a.Length != 0 && f.Length == 0)) == 1) { nkitJunkFiles++; } else if ((c = fnd.Count(a => a.Length == f.Length)) != 1) { failCount++; _log?.LogDetail(string.Format("FST Error: File Size bad - {0}/{1} (Length {2}){3}", f.Path, f.Name, f.Length.ToString(), c <= 1 ? "" : string.Format(" {0} files found", c.ToString()))); } else if (f.DataOffset != fnd[0].DataOffset) { filesMoved++; } if (failCount >= 10) { break; } } if (failCount != 0) { if (failCount >= 10) { throw new HandledException(string.Format("{0} or more FST errors found", failCount.ToString())); } } if (filesMoved != 0) { _log?.LogDetail(string.Format("{0} file{1} of {2} will be repositioned when rebuilding this image", filesMoved.ToString(), filesMoved == 1 ? "" : "s", fstFiles.Count.ToString())); } if (nkitJunkFiles != 0) { _log?.LogDetail(string.Format("{0} file{1} of {2} will be generated from junk when rebuilding this image", nkitJunkFiles.ToString(), nkitJunkFiles == 1 ? "" : "s", fstFiles.Count.ToString())); } if (goodFst != null) { //is the data output so far correct if (!bruteForceValidHeader(ctx, goodFst, hdr, bi2bin, appldr, maindol, fst, pc)) { throw new HandledException(string.Format("Post FST Crc Failed 0x{0}", goodFst.PostFstCrc.ToString("X8"))); } } //############################################################################ //# WRITE DISC START CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write); target.Write(hdr.Data, 0, (int)hdr.Size); pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read long dstPos = hdr.Data.Length; crc.Snapshot("boot.bin"); target.Write(bi2bin.Data, 0, (int)bi2bin.Size); dstPos += bi2bin.Size; crc.Snapshot("bi2.bin"); target.Write(appldr.Data, 0, (int)appldr.Size); dstPos += appldr.Size; ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos); dstPos += Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos; crc.Snapshot("appldr.bin"); target.Write(maindol.Data, 0, (int)maindol.Size); dstPos += maindol.Size; if (goodFst != null) { long padding = goodFst.FstOffset - dstPos; ByteStream.Zeros.Copy(target, padding); dstPos += padding; } crc.Snapshot("main.dol"); target.Write(fst.Data, 0, fst.Data.Length); dstPos += fst.Size; bool firstFile = true; Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>(); crc.Snapshot("fst.bin"); //############################################################################ //# WRITE THE FILESYSTEM int fidx = -1; FstFile lastFile = new FstFile(null) { DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = fst.Size }; FstFile nextFile = fstFiles[++fidx]; FstFile cacheFile = null; long nullsPos = dstPos + 0x1c; if (nullsPos % 4 != 0) { nullsPos += 4 - (nullsPos % 4); } //########### FILES foreach (FstFile f in srcFiles) //read the files and write them out as goodFiles (possible order difference) { while ((cacheFile = cache.Keys.FirstOrDefault(a => (a.Length == nextFile.Length || (inStream.IsNkit && a.Length == 0)) && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) //write cache { bool isJunk = inStream.IsNkit && cacheFile.Length == 0 && cacheFile.Name == nextFile.Name && cacheFile.Path == nextFile.Path; using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile])) { dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, isJunk); } cache.Remove(cacheFile); lastFile = nextFile; nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null; } if (f.DataOffset - srcPos > 0) //skip src junk { inStream.Copy(ByteStream.Zeros, f.DataOffset - srcPos); srcPos += f.DataOffset - srcPos; } if (nextFile != null && (f.Length != nextFile.Length || f.Name != nextFile.Name || f.Path != nextFile.Path)) //cache file (nkit junk files are cached) { bool isNkitJunk = inStream.IsNkit && f.Length == 0 && nextFile.Length != 0 && f.Name == nextFile.Name && f.Path == nextFile.Path; byte[] cacheItem = new byte[isNkitJunk ? 8 : f.Length]; inStream.Read(cacheItem, 0, cacheItem.Length); //read the nkit junk data - details (real length and null counts) cache.Add(f, cacheItem); srcPos += cacheItem.Length; } else //copy file { dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, inStream, lastFile, nextFile, patches, inStream, junkStart, false); lastFile = nextFile; nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null; srcPos += f.Length; } } while (nextFile != null && fidx < fstFiles.Count && (cacheFile = cache.Keys.FirstOrDefault(a => a.Length == nextFile.Length && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) { using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile])) { dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, false); } cache.Remove(cacheFile); lastFile = nextFile; nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null; } writeDestGap(nullsPos, target, dstPos, pc.OutputSize - dstPos, true, patches, junkStart, inStream); crc.Snapshot("files"); } resultMsg = "MatchFail"; uint finalCrc = crc.FullCrc(true); if (ctx.Dats.RedumpData.FirstOrDefault(a => a.Crc == finalCrc) != null) { resultMsg = "Match Redump"; } if (ctx.Dats.CustomData.FirstOrDefault(a => a.Crc == finalCrc) != null) { resultMsg = "Match Custom"; } pc.ReaderCheckPoint2Complete(crc, false, finalCrc, finalCrc, VerifyIsWrite, hdr.Data, resultMsg); pc.ReaderCheckPoint3Complete(); } catch (Exception ex) { throw pc.SetReaderException(ex, "RestoreReaderGc.Read - Read and repair"); //don't let the writer lock } }
public void SeekToFile(FstFile file) { _seek = file.Offset - (file.Offset % GroupSize); //offset within partition of group, set to group boundary }
public override string ToString() { return(string.Format("{0} : {1} : {2}", FstFile.ToString(), GapLength.ToString("X8"), Alignment.ToString())); }
internal static FileSystem Parse(MemorySection ms, FstFile fst, string id) { return(Parse(ms, fst, id, false)); }
internal static long CopyFile(ref long nullsPos, ConvertFile conFile, FstFile prevFile, Stream dest, ref long srcPos, long dstPos, Stream srcStream, JunkStream junkNStream, long imageSize, out bool missing) { //long pos = dest.Position; missing = false; FstFile file = conFile.FstFile; long size = file.Length + (file.Length % 4 == 0 ? 0 : (4 - (file.Length % 4))); if (srcPos + size > imageSize) { size = file.Length; //some rare GC demos end at the end of a non aligned file. This fixes it - v1.2 bugfix } long written = size; byte[] f = new byte[Math.Min(0x30, size)]; srcStream.Read(f, 0, f.Length); //then read while junk is created if (prevFile != null && prevFile.DataOffset == file.DataOffset && prevFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size { nullsPos = srcPos + 0x1CL; //will already be aligned } int nulls = (int)(nullsPos - srcPos); int countNulls = 0; if (f.Length > nulls) { junkNStream.Position = file.DataOffset; //async junk gen for (int i = 0; i < f.Length && f[i] == 0; i++) { countNulls++; } if (countNulls < f.Length) //don't test all nulls { missing = junkNStream.Compare(f, 0, f.Length, Math.Max(0, countNulls)) == f.Length; } } if (missing) //start of file is junk { //check the remainder of the file MemorySection junkFile = MemorySection.Read(srcStream, size - f.Length); missing = junkNStream.Compare(junkFile.Data, 0, (int)junkFile.Size, 0) == junkFile.Size; if (missing) { written = 0; conFile.Gap.SetJunkFile((uint)conFile.FstFile.Length, countNulls); } else //not 100% junk, write the file out { dest.Write(f, 0, f.Length); dest.Write(junkFile.Data, 0, (int)junkFile.Size); } junkFile = null; } else { dest.Write(f, 0, f.Length); srcStream.Copy(dest, size - f.Length); //copy file } if (!missing) //reset the gap when no junk { nullsPos = srcPos + size + 0x1c; if (nullsPos % 4 != 0) { nullsPos += 4 - (nullsPos % 4); } } srcPos += size; return(written); }
internal static List <ConvertFile> GetConvertFstFiles(Stream inStream, long size, MemorySection hdr, MemorySection fst, bool isGc, long fstFileAlignment, out string error) { string[] align = new string[] { ".tgc" }; //, ".dol", ".thp", ".adp", ".poo", ".pcm", ".pcm16", ".son", ".act", ".bin", ".gpl", ".tpl", ".bmp", ".sam", ".sdi", ".tcs", ".txt", ".skn", ".stp", ".anm" }; error = null; List <ConvertFile> conFiles = new List <ConvertFile>(); try { List <FstFile> srcFiles = FileSystem.Parse(fst, null, hdr.ReadString(0, 4), isGc)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList(); //get list of files and gaps long end; long gap; long fstLen = hdr.ReadUInt32B(0x424) * (isGc ? 1L : 4L); FstFile ff = new FstFile(null) { Name = "fst.bin", DataOffset = fstLen, Offset = fstLen, Length = (int)fst.Size, IsNonFstFile = true }; for (int i = 0; i < srcFiles.Count; i++) { ff = i == 0 ? ff : srcFiles[i - 1]; end = ff.DataOffset + ff.Length; end += end % 4 == 0 ? 0 : 4 - (end % 4); gap = srcFiles[i].DataOffset - end; if (gap < 0) { error = string.Format("The gap between '{0}' and '{1}' is {2} - Converting as bad image", ff.Name, srcFiles[i].Name, gap.ToString()); return(null); } conFiles.Add(new ConvertFile(gap, isGc) { FstFile = ff }); } ff = srcFiles.Last(); end = ff.DataOffset + ff.Length; end += end % 4 == 0 ? 0 : 4 - (end % 4); gap = size - end; if (gap >= -3 && gap < 0) { gap = 0; //some hacked gc images converted from tgc end on the file end (star fox e3) } if (gap < 0) { error = string.Format("The gap between '{0}' and the end of the image is {1} - Converting as bad image/partition", ff.Name, gap.ToString()); return(null); } conFiles.Add(new ConvertFile(gap, isGc) { FstFile = ff }); //set alignment foreach (ConvertFile cf in conFiles) { ff = cf.FstFile; if (fstFileAlignment == 0) { cf.Alignment = 0; //preserve alignment } else if (fstFileAlignment == -1 && ff.DataOffset % 0x8000 == 0 && (ff.Length % 0x8000 == 0 || align.Contains(Path.GetExtension(ff.Name).ToLower()))) { cf.Alignment = 0x8000; //default behaviour } else if (fstFileAlignment != 0 && ff.DataOffset % fstFileAlignment == 0) //src matches alignment { cf.Alignment = fstFileAlignment; //align to largest multiple } else { cf.Alignment = -1; //-1 = do not align this file } } } catch { error = "Fst parsing error - Converting as bad image"; return(null); } return(conFiles); }
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc) { try { DatData data = ctx.Dats; List <string> addedFiles = new List <string>(); DateTime dt = DateTime.Now; MemorySection hdr = inStream.DiscHeader; string idVer = hdr.ReadString(0x200, 8); if (idVer != "NKIT v01") { throw new Exception(string.Format("{0} not supported by this version", idVer)); } bool isNkit = idVer.StartsWith("NKIT"); uint nkitCrc = hdr.ReadUInt32B(0x208); uint imageSize = hdr.ReadUInt32B(0x210); string junkId = hdr.ReadString(0x214, 4); if (junkId != "\0\0\0\0") { inStream.ChangeJunk(junkId); } hdr.WriteUInt32B(0x200, 0); hdr.WriteUInt32B(0x204, 0); hdr.WriteUInt32B(0x208, 0); hdr.WriteUInt32B(0x20C, 0); hdr.WriteUInt32B(0x210, 0); hdr.WriteUInt32B(0x214, 0); hdr.WriteUInt32B(0x218, 0); MemorySection fst; long mainDolAddr = hdr.ReadUInt32B(0x420); NCrc crc = new NCrc(); NStream target = new NStream(new CryptoStream(outStream, crc, CryptoStreamMode.Write)); //############################################################################ //# READ DISC START MemorySection hdrToFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - hdr.Size); fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) + (hdr.ReadUInt32B(0x428) % 4 == 0 ? 0 : 4 - (hdr.ReadUInt32B(0x428) % 4))); long srcPos = hdr.ReadUInt32B(0x424) + fst.Size; //############################################################################ //# WRITE DISC START target.Write(hdr.Data, 0, (int)hdr.Size); pc.ReaderCheckPoint1PreWrite(junkId, hdr.ReadUInt32B(0x208)); //size that we will output from this read crc.Snapshot("hdr.bin"); target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read crc.Snapshot("bi2.bin, appldr.bin, main.dol"); target.Write(fst.Data, 0, fst.Data.Length); crc.Snapshot("fst.bin"); hdrToFst = null; //let this be collected if needed long dstPos = hdr.ReadUInt32B(0x424) + fst.Size; long nullsPos = dstPos + 0x1c; string error; List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, inStream.Length, hdr, fst, true, -1, out error); //result.ImageInfo.IsoSize if (conFiles == null) { if (error != null) { _log?.LogDetail(error); } ConvertFile cf = new ConvertFile(inStream.Length - srcPos, true) //result.ImageInfo.IsoSize { FstFile = new FstFile(null) { DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size }, }; dstPos += writeGap(cf, ref nullsPos, ref srcPos, dstPos, inStream, target, true); } else { //########### FILES bool firstFile = true; for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference { ConvertFile f = conFiles[i]; FstFile ff = f.FstFile; if (!firstFile) //fst already written { //Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), dstPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (dstPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name)); if (srcPos < ff.DataOffset) { inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc srcPos += ff.DataOffset - srcPos; } //write file if (ff.DataOffset == mainDolAddr) { hdr.WriteUInt32B(0x420, (uint)dstPos); } fst.WriteUInt32B(ff.OffsetInFstFile, (uint)dstPos); dstPos += copyFile(f, ref nullsPos, ref srcPos, dstPos, imageSize, inStream, target); } if (dstPos < imageSize) { dstPos += writeGap(f, ref nullsPos, ref srcPos, dstPos, inStream, target, i == 0 || i == conFiles.Count - 1); if (!firstFile) { fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)ff.Length); } } firstFile = false; } } crc.Snapshot("files"); crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); crc.Crcs[0].PatchData = hdr.Data; crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data); crc.Crcs[2].PatchData = fst.Data; if (imageSize != dstPos) { throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString())); } pc.ReaderCheckPoint2Complete(crc, false, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid"); pc.ReaderCheckPoint3Complete(); } catch (Exception ex) { throw pc.SetReaderException(ex, "NkitReaderGc.Read - Read and convert"); //don't let the writer lock } }
private long partitionStreamWrite(LongRef outSize, Stream inStream, Stream target, long size, DatData settingsData, NkitPartitionPatchInfo patchInfo, WiiHashStore hashes, Coordinator pc) { DatData data = settingsData; List <string> addedFiles = new List <string>(); DateTime dt = DateTime.Now; MemorySection hdr = MemorySection.Read(inStream, 0x440); long srcPos = hdr.Size; long outPos = 0; long imageSize = 0; try { if (hdr.ReadString(0, 4) == "\0\0\0\0") { long nullsPos = 0; long fileLength = -1; LongRef gapLength = new LongRef() { Value = -1 }; target.Write(hdr.Data, 0, (int)hdr.Size); MemorySection sz = MemorySection.Read(inStream, 4); srcPos += 4; outPos += hdr.Size; imageSize = sz.ReadUInt32B(0) * 4L; outSize.Value = NStream.HashedLenToData(imageSize); JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), outSize.Value); //SET LENGTH FROM HEADER outPos += writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager); } else { string idVer = hdr.ReadString(0x200, 8); if (idVer != "NKIT v01") { throw new Exception(string.Format("{0} not supported by this version", idVer)); } bool isNkit = idVer.StartsWith("NKIT"); imageSize = NStream.HashedLenToData((hdr.ReadUInt32B(0x210) * 4L)); outSize.Value = imageSize; string junkId = hdr.ReadString(0x214, 4); JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), imageSize); //SET LENGTH FROM HEADER MemorySection fst; long mainDolAddr = hdr.ReadUInt32B(0x420); //############################################################################ //# READ DISC START MemorySection hdrToFst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x424) * 4L) - hdr.Size); srcPos += hdrToFst.Size; fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) * 4L); long postFstPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size; srcPos += fst.Size; hashes.WriteFlagsData(imageSize, inStream); srcPos += hashes.FlagsLength; patchInfo.PartitionDataHeader = hdr; patchInfo.Fst = fst; //############################################################################ //# WRITE DISC START target.Write(hdr.Data, 0, (int)hdr.Size); target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read target.Write(fst.Data, 0, fst.Data.Length); hdrToFst = null; //let this be collected if needed outPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size; long nullsPos = outPos + 0x1c; string error; List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, size, hdr, fst, false, -1, out error); if (conFiles == null) { if (error != null) { _log?.LogDetail(error); } ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //result.ImageInfo.IsoSize { FstFile = new FstFile(null) { DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size }, }; outPos += writeGap(cf, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager); } else { conFiles[0].GapLength -= hashes.FlagsLength; //fix for a few customs (no gap between the fst and the first file on the source image, but the hash mask makes it look like there is) //########### FILES bool firstFile = true; for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference { ConvertFile f = conFiles[i]; FstFile ff = f.FstFile; if (!firstFile) //fst already written { //Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), outPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (outPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name)); if (srcPos < ff.DataOffset) //skip any padding (not written for wii currently) { inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc srcPos += ff.DataOffset - srcPos; } //write file if (ff.DataOffset == mainDolAddr) { hdr.WriteUInt32B(0x420, (uint)(outPos / 4L)); } fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(outPos / 4L)); outPos += copyFile(f, ref nullsPos, ref srcPos, outPos, inStream, target); } if (outPos < imageSize) { long gapLen = writeGap(f, ref nullsPos, ref srcPos, outPos, inStream, target, junk, i == 0 || i == conFiles.Count - 1, patchInfo.ScrubManager); outPos += gapLen; if (!firstFile) { fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)(ff.Length)); } } firstFile = false; } } } return(srcPos); } catch (Exception ex) { throw pc.SetReaderException(ex, "NkitReaderWii.Read - partitionRead"); } }