public void Populate()
{
if (!string.IsNullOrEmpty(this.Filename) && File.Exists(this.Filename))
{
MemorySection ms = new MemorySection(File.ReadAllBytes(this.Filename));
this.MainDolOffset = ms.ReadUInt32B(0x00);
this.FstOffset = ms.ReadUInt32B(0x04);
this.MaxFst = ms.ReadUInt32B(0x08);
this.Region = (Region)ms.ReadUInt32B(0x0C);
this.Title = ms.Read(0x10, 0x50 - 0x10);
this.FstData = ms.Read(0x50, (int)ms.Size - 0x50);
}
}
public static FileSystem Parse(Stream fstData, long fstOffset, long length, string id, bool isGc)
{
MemorySection ms = MemorySection.Read(fstData, length);
FstFile ff = new FstFile(null)
{
Name = "fst.bin", DataOffset = fstOffset, Offset = NStream.DataToOffset(fstOffset, !isGc), IsNonFstFile = true, Length = (int)fstData.Length
};
return(Parse(ms, ff, id, isGc));
}
public HeaderBruteForcer(uint[] updateCrcs, SortedList <uint, bool> checkCrcs, Tuple <byte[], int[]>[] regionData, params byte[][] headers)
{
_updateCrcs = updateCrcs;
_checkCrcs = checkCrcs;
//DateTime dt = DateTime.Now;
_hdrs = headers.Select(a => new headerCrc()
{
Crc = Crc.Compute(a), CrcPreRegion = Crc.Compute(a, 0, 0x4e000), CrcRegion = Crc.Compute(a, 0x4e000, 0x20), CrcPostRegion = Crc.Compute(a, 0x4e020, 0x1fe0), Data = a
}).ToList();
MemorySection ms = new MemorySection(headers[0]);
_origRegion = (int)ms.ReadUInt32B(0x4e000);
_origRegionData = ms.Read(0x4e010, 0x10);
_origHeaderCrcs = _hdrs.Select(a => a.Crc).ToArray(); //original header Crcs
for (int h = _hdrs.Count - 1; h >= 1; h--)
{
if (_hdrs[h].Crc == _hdrs[h - 1].Crc)
{
_hdrs.RemoveAt(h); //remove duplicate headers
}
}
_regionData = new List <regionCrc>();
for (int h = 0; h < _hdrs.Count; h++)
{
for (int i = 0; i < regionData.Length; i++)
{
for (int r = 0; r < regionData[i].Item2.Length; r++)
{
_regionData.Add(new regionCrc()
{
Crc = 0, Header = _hdrs[h], RegionData = regionData[i].Item1, Region = (int)regionData[i].Item2[r]
});
}
}
}
//parallel precalculate the crcs for the matching
Parallel.ForEach(_regionData, rgn =>
{
MemorySection rg = new MemorySection(new byte[0x20]);
rg.WriteUInt32B(0, (uint)rgn.Region);
rg.Write(0x10, rgn.RegionData);
rgn.Crc = ~Crc.Combine(Crc.Combine(~rgn.Header.CrcPreRegion, ~Crc.Compute(rg.Data), 0x20), ~rgn.Header.CrcPostRegion, 0x1fe0);
});
//Trace.WriteLine(string.Format("Crc Headers: {0} - Took: {1}", _regionData.Count.ToString(), (DateTime.Now - dt).TotalMilliseconds.ToString()));
}
public ExtractResult ExtractBasicInfo()
{
Region region;
if (this.IsGameCube)
{
MemorySection bi2bin = MemorySection.Read(this.NStream, 0x2000);
region = (Region)bi2bin.ReadUInt32B(0x18);
}
else
{
region = (Region)this.Header.ReadUInt32B(0x4e000);
}
return(createExtractResult(region, null));
}
private ExtractResult ExtractFilesWiiNkit(Func <ExtractedFile, bool> filter, Action <Stream, ExtractedFile> extract)
{
WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)Header;
ExtractedFile exFile = new ExtractedFile(IsGameCube ? DiscType.GameCube : DiscType.Wii,
NStream.Id8, null, 0, hdr.Size, "", "hdr.bin", ExtractedFileType.WiiDiscItem);
long srcPos = hdr.Size;
if (filter(exFile))
{
using (MemoryStream ms = new MemoryStream(hdr.Data))
{
extract(ms, exFile);
}
}
foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted
{
if (NStream.Position < part.DiscOffset)
{
NStream.Copy(Stream.Null, part.DiscOffset - NStream.Position);
}
long prtPos = NStream.Position;
MemorySection prtHdr = MemorySection.Read(NStream, 0x20000);
MemorySection prtDataHdr = MemorySection.Read(NStream, 0x440);
string prtId = prtDataHdr.ReadString(0, 4);
if (prtId != "\0\0\0\0")
{
srcPos += prtHdr.Size + prtDataHdr.Size;
exFile = new ExtractedFile(IsGameCube ? DiscType.GameCube : DiscType.Wii,
NStream.Id8, null, prtPos, prtHdr.Size, "", prtId + "hdr.bin", ExtractedFileType.WiiDiscItem);
if (filter(exFile))
{
using (MemoryStream ms = new MemoryStream(prtHdr.Data))
{
extract(ms, exFile);
}
}
extractFiles(prtId, prtDataHdr, NStream, filter, extract);
}
}
return(createExtractResult((Region)Header.ReadUInt32B(0x4e000), null));
}
private void setDiscHeader(bool autodetect, long[] isoDecParts)
{
if (autodetect)
{
//4 bytes have already been read
//read the next part of the header
byte[] x = new byte[0x20];
_id.CopyTo(x, 0);
this.Read(x, 4, 0x20 - 4);
_isWii = x[0x18] == 0x5d && x[0x19] == 0x1c && x[0x1a] == 0x9e && x[0x1b] == 0xa3;
_isGamecube = x[0x1c] == 0xc2 && x[0x1d] == 0x33 && x[0x1e] == 0x9f && x[0x1f] == 0x3d;
if (!_isWii && !_isGamecube)
{
_isGamecube = true; //v1.1 BugFix for Dodger Demo_shrunk.gcm
}
byte[] h = new byte[_isWii ? _HeaderSizeWii : _HeaderSizeGc];
x.CopyTo(h, 0);
this.Read(h, x.Length, h.Length - x.Length);
this.DiscHeader = new MemorySection(h);
}
else
{
this.DiscHeader = MemorySection.Read(this, _isWii ? _HeaderSizeWii : _HeaderSizeGc);
}
if (this.IsWii)
{
this.DiscHeader = new WiiDiscHeaderSection(this.DiscHeader);
((WiiDiscHeaderSection)this.DiscHeader).IsoDecPartitions = isoDecParts;
}
_isNkit = this.DiscHeader.ReadString(0x200, 4) == "NKIT";
_isNkitUpdateRemoved = this.DiscHeader.ReadUInt32B(0x218) != 0; //update not 0means the update partition has been removed
if (_isNkit)
{
_imageSize = this.DiscHeader.ReadUInt32B(0x210) * (_isWii ? 4L : 1L);
}
_position = this.DiscHeader.Size;
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
DatData data = ctx.Dats;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
uint imageSize = hdr.ReadUInt32B(0x210);
string junkId = hdr.ReadString(0x214, 4);
if (junkId != "\0\0\0\0")
{
inStream.ChangeJunk(junkId);
}
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
NCrc crc = new NCrc();
NStream target = new NStream(new CryptoStream(outStream, crc, CryptoStreamMode.Write));
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - hdr.Size);
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) + (hdr.ReadUInt32B(0x428) % 4 == 0 ? 0 : 4 - (hdr.ReadUInt32B(0x428) % 4)));
long srcPos = hdr.ReadUInt32B(0x424) + fst.Size;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(junkId, hdr.ReadUInt32B(0x208)); //size that we will output from this read
crc.Snapshot("hdr.bin");
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
crc.Snapshot("fst.bin");
hdrToFst = null; //let this be collected if needed
long dstPos = hdr.ReadUInt32B(0x424) + fst.Size;
long nullsPos = dstPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, inStream.Length, hdr, fst, true, -1, out error); //result.ImageInfo.IsoSize
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(inStream.Length - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += writeGap(cf, ref nullsPos, ref srcPos, dstPos, inStream, target, true);
}
else
{
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), dstPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (dstPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)dstPos);
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)dstPos);
dstPos += copyFile(f, ref nullsPos, ref srcPos, dstPos, imageSize, inStream, target);
}
if (dstPos < imageSize)
{
dstPos += writeGap(f, ref nullsPos, ref srcPos, dstPos, inStream, target, i == 0 || i == conFiles.Count - 1);
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)ff.Length);
}
}
firstFile = false;
}
}
crc.Snapshot("files");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, false, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderGc.Read - Read and convert"); //don't let the writer lock
}
}
public void WriteFlagsData(long partitionDataSize, Stream readStream)
{
_flags = MemorySection.Read(readStream, intsCount(partitionDataSize) * 4);
}
private long partitionWrite(Stream inStream, NCrc crc, Stream target, WiiPartitionSection pHdr, Context ctx, Coordinator pc, NkitInfo imageInfo, ScrubManager scrub, WiiHashStore hashes, long fstFileAlignment)
{
#if DEHASH
return(pHdr.NewPartitionDataLength = inStream.Copy(target, pHdr.PartitionDataLength, null));
#elif DECRYPT
inStream.Copy(target, pHdr.PartitionLength, null);
return(pHdr.NewPartitionDataLength = pHdr.PartitionDataLength);
#endif
MemorySection hdr = MemorySection.Read(inStream, 0x440);
//ProgressResult result = ctx.Result;
long mlt = 4L; //GC 1L
long logicalSize = pHdr.PartitionDataLength; //GC: ctx.ImageLength;
long physicalSize = pHdr.PartitionDataLength; //GC: result.ImageInfo.IsoSize;
string junkId = hdr.ReadString(0, 4);
//bool write = true;
List <string> addedFiles = new List <string>();
long srcPos;
long dstPos = 0;
JunkStream js = new JunkStream(junkId, hdr.Read8(6), logicalSize);
try
{
if (junkId == "\0\0\0\0")
{
srcPos = hdr.Size;
imageInfo.BytesData = srcPos;
_log?.LogDetail("Null Partition ID, preserving partition as raw");
ConvertFile cf = new ConvertFile(logicalSize - hdr.Size, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.Size, Offset = hdr.Size, Length = 0
},
};
long nullsPos = 0;
target.Write(hdr.Data, 0, (int)hdr.Size); //0x400
target.Write(pHdr.Header.Data, 0x2bc, 4); //copy the original partition length
dstPos += 0x440 + 4 + NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
hdr.WriteString(0x200, 8, "NKIT v01");
hdr.WriteUInt32B(0x210, (uint)(pHdr.PartitionLength / mlt));
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
crc.Snapshot(junkId + " PrtHdr");
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot(junkId + " Fst");
target.Write(hashes.FlagsToByteArray(), 0, (int)hashes.FlagsLength);
crc.Snapshot(junkId + " HashFlags");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos + hashes.FlagsLength;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, physicalSize, hdr, fst, false, fstFileAlignment, out error);
imageInfo.BytesData = srcPos;
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(pHdr.PartitionDataLength - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
List <ConvertFile> missing;
NkitFormat.NkitWriteFileSystem(ctx, imageInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out missing, scrub, pHdr.PartitionDataLength, _log);
dstPos += hashes.HashesToStream(target);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
crc.Crcs[crc.Crcs.Length - 1].PatchData = hashes.FlagsToByteArray();
crc.Crcs[crc.Crcs.Length - 1].PatchCrc = Crc.Compute(crc.Crcs[crc.Crcs.Length - 2].PatchData);
crc.Crcs[crc.Crcs.Length - 2].PatchData = fst.Data;
crc.Crcs[crc.Crcs.Length - 2].PatchCrc = Crc.Compute(fst.Data);
}
}
}
catch (Exception ex)
{
throw new HandledException(ex, "NkitWriterGc.Write - Convert");
}
return(dstPos);
}
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
{
try
{
long mlt = 1L; //for Wii: 4L
long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength
pc.WriterCheckPoint1WriteReady(out string junkId); //wait until read has written the header and set the length
List <string> addedFiles = new List <string>();
NCrc crc = new NCrc();
long srcPos;
long dstPos = 0;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
string id8 = string.Concat(hdr.ReadString(0, 6), hdr.Data[6].ToString("X2"), hdr.Data[7].ToString("X2"));
if (junkId == null)
{
junkId = ctx.Settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == id8)?.JunkId;
if (junkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", junkId, id8.Substring(0, 4)));
}
}
if (junkId == null)
{
junkId = hdr.ReadString(0, 4);
}
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
long fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == id8)?.Item2 ?? -1;
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
crc.Snapshot("hdr.bin");
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot("fst.bin");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
JunkStream js = new JunkStream(junkId, hdr.Read8(6), NStream.FullSizeGameCube);
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, imageSize, hdr, fst, true, fstFileAlignment, out string error); //Size isn't important for writing //result.ImageInfo.IsoSize
NkitInfo nkitInfo = new NkitInfo
{
BytesData = srcPos,
BytesGaps = 0,
BytesJunkFiles = 0,
BytesPreservationData = 0,
BytesPreservationDiscPadding = 0
};
ScrubManager scrub = new ScrubManager();
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
NkitFormat.NkitWriteFileSystem(ctx, nkitInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out List <ConvertFile> missing, scrub, imageSize, _log);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
}
if (dstPos % 0x800 != 0)
{
long l = 0x800 - (dstPos % 0x800);
ByteStream.Zeros.Copy(target, l);
dstPos += l;
nkitInfo.BytesPreservationDiscPadding += l;
}
crc.Snapshot("files");
NkitFormat.LogNkitInfo(nkitInfo, _log, hdr.ReadString(0, 4), true);
pc.WriterCheckPoint2Complete(out NCrc readerCrcs, out uint validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.
hdr.WriteUInt32B(0x420, (uint)mainDolAddr);
hdr.WriteString(0x200, 8, "NKIT v01"); //header and version
hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
hdr.WriteUInt32B(0x210, (uint)imageSize); //result.ImageInfo.IsoSize);
hdr.WriteString(0x214, 4, hdr.ReadString(0, 4) != junkId ? junkId : "\0\0\0\0");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, "NKit Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "NkitWriterGc.Write - Convert");
}
}
internal static long CopyFile(ref long nullsPos, ConvertFile conFile, FstFile prevFile, Stream dest, ref long srcPos, long dstPos, Stream srcStream, JunkStream junkNStream, long imageSize, out bool missing)
{
//long pos = dest.Position;
missing = false;
FstFile file = conFile.FstFile;
long size = file.Length + (file.Length % 4 == 0 ? 0 : (4 - (file.Length % 4)));
if (srcPos + size > imageSize)
{
size = file.Length; //some rare GC demos end at the end of a non aligned file. This fixes it - v1.2 bugfix
}
long written = size;
byte[] f = new byte[Math.Min(0x30, size)];
srcStream.Read(f, 0, f.Length); //then read while junk is created
if (prevFile != null && prevFile.DataOffset == file.DataOffset && prevFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size
{
nullsPos = srcPos + 0x1CL; //will already be aligned
}
int nulls = (int)(nullsPos - srcPos);
int countNulls = 0;
if (f.Length > nulls)
{
junkNStream.Position = file.DataOffset; //async junk gen
for (int i = 0; i < f.Length && f[i] == 0; i++)
{
countNulls++;
}
if (countNulls < f.Length) //don't test all nulls
{
missing = junkNStream.Compare(f, 0, f.Length, Math.Max(0, countNulls)) == f.Length;
}
}
if (missing) //start of file is junk
{
//check the remainder of the file
MemorySection junkFile = MemorySection.Read(srcStream, size - f.Length);
missing = junkNStream.Compare(junkFile.Data, 0, (int)junkFile.Size, 0) == junkFile.Size;
if (missing)
{
written = 0;
conFile.Gap.SetJunkFile((uint)conFile.FstFile.Length, countNulls);
}
else //not 100% junk, write the file out
{
dest.Write(f, 0, f.Length);
dest.Write(junkFile.Data, 0, (int)junkFile.Size);
}
junkFile = null;
}
else
{
dest.Write(f, 0, f.Length);
srcStream.Copy(dest, size - f.Length); //copy file
}
if (!missing) //reset the gap when no junk
{
nullsPos = srcPos + size + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
}
srcPos += size;
return(written);
}
public ExtractResult ExtractRecoveryFilesGc()
{
MemorySection hdr = NStream.DiscHeader;
MemorySection bi2bin;
List <ExtractRecoveryResult> result = new List <ExtractRecoveryResult>();
NCrc crc = new NCrc();
uint appLdrCrc = 0;
MemorySection fst = null;
uint fstCrc = 0;
string fn;
string tmpFullName;
Log?.Log(string.Format("Processing: {0}", SourceFileName));
int storeType;
using (CryptoStream bw = new CryptoStream(ByteStream.Zeros, crc, CryptoStreamMode.Write))
{
bw.Write(hdr.Data, 0, (int)hdr.Size);
bi2bin = MemorySection.Read(NStream, 0x2000);
bw.Write(bi2bin.Data, 0, (int)bi2bin.Size);
MemorySection aplHdr = MemorySection.Read(NStream, 0x20);
byte[] appLdr = new byte[0x20 + aplHdr.ReadUInt32B(0x14) + aplHdr.ReadUInt32B(0x18)];
Array.Copy(aplHdr.Data, appLdr, (int)aplHdr.Size);
NStream.Read(appLdr, (int)aplHdr.Size, appLdr.Length - (int)aplHdr.Size);
appLdrCrc = Crc.Compute(appLdr);
fn = string.Format("appldr[{0}][{1}].bin", aplHdr.ReadString(0, 10).Replace("/", ""), appLdrCrc.ToString("X8"));
Log?.Log(string.Format(" 1 of 2 - Extracting appldr.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + appLdrCrc.ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, appLdr);
if ((storeType = storeRecoveryFile(this.Settings.RedumpAppldrCrcs.Contains(appLdrCrc), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
bw.Write(appLdr, 0, appLdr.Length); //add to fullcrc
NStream.Copy(bw, hdr.ReadUInt32B(0x424) - (0x2440 + appLdr.Length));
byte[] fstData = new byte[(int)((4 * 4) + (0x60 - 0x20) + hdr.ReadUInt32B(0x428))];
fst = new MemorySection(fstData); //info: dolAddr, fstAddr, maxfst, region, title, fst
NStream.Read(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
fst.WriteUInt32B(0x00, hdr.ReadUInt32B(0x420)); //dol
fst.WriteUInt32B(0x04, hdr.ReadUInt32B(0x424)); //fstAddr
fst.WriteUInt32B(0x08, hdr.ReadUInt32B(0x42C)); //maxfst
fst.WriteUInt32B(0x0c, bi2bin.ReadUInt32B(0x18)); //region
fst.Write(0x10, hdr.Read(0x20, 0x60 - 0x20)); //title
fstCrc = Crc.Compute(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428)); //fst
bw.Write(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
crc.Snapshot("crc");
}
//fst checksums are postfst
fn = string.Format("fst[{0}][{1}][{2}][{3}].bin", NStream.Id8, appLdrCrc.ToString("X8"), fstCrc.ToString("X8"), crc.FullCrc().ToString("X8"));
Log?.Log(string.Format(" 2 of 2 - Extracting fst.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + crc.FullCrc().ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, fst.Data);
if ((storeType = storeRecoveryFile(this.Settings.RedumpFstCrcs.Contains(crc.FullCrc()), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
return(createExtractResult((Region)bi2bin.Read8(0x18), result.ToArray()));
}
private ExtractResult extractFiles(string id, MemorySection hdr, Stream inStream, Func <ExtractedFile, bool> filter, Action <Stream, ExtractedFile> extract)
{
long mlt = this.IsGameCube ? 1L : 4L;
long srcPos = 0;
List <FstFile> files = new List <FstFile>();
MemorySection bootbin = (this.IsGameCube || this.NStream.IsNkit) ? hdr: MemorySection.Read(inStream, 0x440);
files.Add(new FstFile(null)
{
Name = "boot.bin", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = (int)bootbin.Size, IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += bootbin.Size;
MemorySection bi2bin = MemorySection.Read(inStream, 0x2000);
files.Add(new FstFile(null)
{
Name = "bi2.bin", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = (int)bi2bin.Size, IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += bi2bin.Size;
//########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol
MemorySection appldr = MemorySection.Read(inStream, Math.Min(bootbin.ReadUInt32B(0x420) == 0 ? uint.MaxValue : (bootbin.ReadUInt32B(0x420) * mlt), (bootbin.ReadUInt32B(0x424) * mlt)) - srcPos);
files.Add(new FstFile(null)
{
Name = "appldr.bin", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = 0x20 + appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18), IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += appldr.Size;
//########### APP (main.dol)
MemorySection maindol = null;
if (bootbin.ReadUInt32B(0x420) < bootbin.ReadUInt32B(0x424))
{
maindol = MemorySection.Read(inStream, (bootbin.ReadUInt32B(0x424) * mlt) - srcPos);
uint maindolSize = 0;
for (int i = 0; i < 18; i++)
{
if (maindol.ReadUInt32B(0x0 + (i * 4)) != 0) //7 text offsets, 11 data offsets
{
maindolSize = Math.Max(maindolSize, maindol.ReadUInt32B(0x0 + (i * 4)) + maindol.ReadUInt32B(0x90 + (i * 4)));
}
}
files.Add(new FstFile(null)
{
Name = "main.dol", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = maindolSize, IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += maindol.Size;
}
MemorySection fstbin = MemorySection.Read(inStream, bootbin.ReadUInt32B(0x428) * mlt);
files.Add(new FstFile(null)
{
Name = "fst.bin", PartitionId = id, DataOffset = srcPos, Offset = srcPos, Length = (int)fstbin.Size, IsNonFstFile = true, OffsetInFstFile = 0
});
srcPos += fstbin.Size;
List <FstFile> fstFiles = FileSystem.Parse(fstbin, null, id, this.IsGameCube)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
if (fstFiles == null)
{
throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", (bootbin.ReadUInt32B(0x424) * mlt).ToString("X8")));
}
files.AddRange(fstFiles);
Dictionary <string, MemorySection> mem = new Dictionary <string, MemorySection>();
mem.Add("boot.bin", bootbin);
mem.Add("bi2.bin", bi2bin);
mem.Add("appldr.bin", appldr);
mem.Add("main.dol", maindol);
mem.Add("fst.bin", fstbin);
List <ExtractedFile> exfiles = files.OrderBy(a => a.Offset).ThenBy(a => a.Length).Select(a => new ExtractedFile(this.IsGameCube ? DiscType.GameCube : DiscType.Wii,
this.NStream.Id8,
this.IsGameCube ? null : id,
a.DataOffset,
a.Length,
a.Path,
a.Name,
mem.ContainsKey(a.Name) ? ExtractedFileType.System : ExtractedFileType.File)).Where(a => filter(a)).ToList();
if (files == null || files.Count == 0)
{
return(createExtractResult((Region)bi2bin.ReadUInt32B(0x18), null));
}
foreach (ExtractedFile f in exfiles) //read the files and write them out as goodFiles (possible order difference)
{
if (srcPos < f.Offset)
{
inStream.Copy(Stream.Null, f.Offset - srcPos);
srcPos += f.Offset - srcPos;
}
if (f.Type == ExtractedFileType.System)
{
using (MemoryStream ms = new MemoryStream(mem[f.Name].Data))
extract(ms, f);
}
else
{
extract(inStream, f);
srcPos += f.Length;
}
}
return(createExtractResult((Region)bi2bin.Read8(0x18), null));
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
string resultMsg = "";
if (!Settings.ConfigFileFound)
{
_log?.Log("!! No config file found - This is required to restore and validate images");
}
NCrc crc = new NCrc();
Settings settings = ctx.Settings;
List <string> addedFiles = new List <string>();
DatData data = ctx.Dats;
RecoveryData rec = ctx.Recovery;
List <JunkRedumpPatch> patches = rec.JunkPatches;
MemorySection hdr = null;
try
{
long junkStart = settings.JunkStartOffsets.FirstOrDefault(a => a.Id8 == inStream.Id8)?.Offset ?? 0;
string forceJunkId = settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == inStream.Id8)?.JunkId;
if (forceJunkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", forceJunkId, inStream.Id));
//result.ImageInfo.JunkId = forceJunkId;
inStream.ChangeJunk(forceJunkId);
}
if (junkStart != 0)
{
_log?.LogDetail(string.Format("Junk forced to start at 0x{0}", junkStart.ToString("X8")));
}
hdr = inStream.DiscHeader;
FstFileItem goodFst = rec.GcBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8 && !(inStream.Id8 == "GNHE5d0000" && a.Length == 116));
if (goodFst != null)
{
_log?.LogDetail(string.Format("Recovery: {0}", goodFst.Filename));
}
if (goodFst == null && rec.GcNewBinFiles != null)
{
goodFst = rec.GcNewBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8);
}
if (goodFst != null)
{
goodFst.Populate();
}
ApploaderFileItem goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
if (goodAldr == null && rec.GcNewBinFiles != null)
{
goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcNewBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
}
//is it an action replay (custom hacks)
if ((hdr.ReadUInt32B(0x420) == 0 && inStream.Id8 == "GNHE5d0000") || inStream.Id8 == "DTLX010000" || inStream.Id8 == "102E010007")
{
if (inStream.Id8 == "102E010007")
{
resultMsg = "Aging Disc detected - Skipping recover";
}
else
{
resultMsg = "Datel Action Replay detected - Skipping recover";
}
try
{
using (CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write))
{
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
inStream.Copy(target, pc.OutputSize - hdr.Size);
}
_log?.LogDetail(resultMsg);
}
catch (Exception ex)
{
throw new HandledException(ex, resultMsg);
}
crc.Snapshot("files");
}
else
{
//############################################################################
//# READ DISC START
//########### Header (boot.bin) 0 (read by base stream already)
long srcPos = hdr.Size;
_log?.LogDetail(string.Format("Header Read: {0} - {1}", friendly(inStream.Id), friendly(inStream.Title)));
//########### Header Info (bi2.bin) 0x440
MemorySection bi2bin = MemorySection.Read(inStream, 0x2000);
srcPos += bi2bin.Size;
//########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol
MemorySection appldr = MemorySection.Read(inStream, Math.Min(hdr.ReadUInt32B(0x420) == 0 ? uint.MaxValue : hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - srcPos);
srcPos += appldr.Size;
//########### APP (main.dol)
MemorySection maindol;
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424))
{
maindol = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - srcPos);
}
else
{
maindol = new MemorySection(new byte[0]);
}
srcPos += maindol.Size;
//########### FST (fst.bin)
MemorySection srcFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428));
srcPos += hdr.ReadUInt32B(0x428);
//############################################################################
//# CORRECT ISSUES
List <FstFile> srcFiles = FileSystem.Parse(srcFst, null, inStream.Id, true)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
if (srcFiles == null)
{
throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", hdr.ReadUInt32B(0x424).ToString("X8")));
}
uint crcTmp = Crc.Compute(appldr.Data, 0, Math.Min((int)appldr.Size, 0x20 + (int)(appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18))));
//adjust appldr
if (goodAldr != null && crcTmp != goodAldr.Crc)
{
_log?.LogDetail(string.Format("Replacing appldr.bin crc {0} with Recovery appldr.bin {1}", crcTmp.ToString("X8"), goodAldr.Crc.ToString("X8")));
addedFiles.Add(goodAldr.Filename);
appldr = new MemorySection(File.ReadAllBytes(goodAldr.Filename));
}
//adjust main.dol
if (goodFst != null)
{
if (goodFst.MainDolOffset != hdr.ReadUInt32B(0x420))
{
//main.dol is before fst in image and after in goodfst
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424) && goodFst.MainDolOffset > goodFst.FstOffset)
{
_log?.LogDetail(string.Format("Skipping main.dol at 0x{0} using 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
maindol = new MemorySection(new byte[0]);
}
else
{
_log?.LogDetail(string.Format("Moving main.dol address 0x{0} to 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
}
hdr.WriteUInt32B(0x420, (uint)goodFst.MainDolOffset);
}
if (goodFst.Region != (Region)bi2bin.ReadUInt32B(0x18))
{
_log?.LogDetail(string.Format("Region Changed to {0} from {1}", goodFst.Region.ToString(), ((Region)bi2bin.ReadUInt32B(0x18)).ToString()));
bi2bin.WriteUInt32B(0x18, (uint)goodFst.Region);
}
if (goodFst.MaxFst != hdr.ReadUInt32B(0x42C))
{
_log?.LogDetail(string.Format("Max Fst Size changed to {0} from {1}", goodFst.MaxFst.ToString("X8"), hdr.ReadUInt32B(0x42C).ToString("X8")));
hdr.WriteUInt32B(0x42C, (uint)goodFst.MaxFst);
}
string newTitle = Encoding.ASCII.GetString(goodFst.Title);
if (newTitle != hdr.ReadString(0x20, 0x60 - 0x20))
{
_log?.LogDetail(string.Format("Title changed to '{0}' from '{1}'", newTitle.TrimEnd('\0'), hdr.ReadString(0x20, 0x60 - 0x20).TrimEnd('\0')));
hdr.Write(0x20, goodFst.Title);
}
}
MemorySection fst = srcFst;
List <FstFile> fstFiles = srcFiles;
crcTmp = Crc.Compute(srcFst.Data);
if (goodFst != null && crcTmp != goodFst.Crc)
{
_log?.LogDetail(string.Format("Replacing fst.bin crc {0} with Recovery fst {1}", crcTmp.ToString("X8"), goodFst.Crc.ToString("X8")));
addedFiles.Add(goodFst.Filename);
fst = new MemorySection(goodFst.FstData);
fstFiles = FileSystem.Parse(fst, null, inStream.Id, true).Files.OrderBy(a => a.Offset).ThenBy(a => a.Length).ToList();
}
//adjust fst.bin
if (goodFst != null && (goodFst.FstOffset != hdr.ReadUInt32B(0x424) || goodFst.FstData.Length != hdr.ReadUInt32B(0x428)))
{
_log?.LogDetail(string.Format("Moving / Resizing fst.bin address 0x{0} (Length {1}) to 0x{2} (Length {3})", hdr.ReadUInt32B(0x424).ToString("X8"), hdr.ReadUInt32B(0x428).ToString(), goodFst.FstOffset.ToString("X8"), goodFst.FstData.Length.ToString()));
hdr.WriteUInt32B(0x424, (uint)goodFst.FstOffset);
hdr.WriteUInt32B(0x428, (uint)goodFst.FstData.Length);
}
int c;
int failCount = 0;
int filesMoved = 0;
int nkitJunkFiles = 0;
foreach (FstFile f in srcFiles)
{
FstFile[] fnd = fstFiles.Where(a => a.Name == f.Name && a.Path == f.Path).ToArray();
if (fnd.Length == 0)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: No File Found - {0}/{1} (Length {2})", f.Path, f.Name, f.Length.ToString()));
}
else if (inStream.IsNkit && (c = fnd.Count(a => a.Length != 0 && f.Length == 0)) == 1)
{
nkitJunkFiles++;
}
else if ((c = fnd.Count(a => a.Length == f.Length)) != 1)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: File Size bad - {0}/{1} (Length {2}){3}", f.Path, f.Name, f.Length.ToString(), c <= 1 ? "" : string.Format(" {0} files found", c.ToString())));
}
else if (f.DataOffset != fnd[0].DataOffset)
{
filesMoved++;
}
if (failCount >= 10)
{
break;
}
}
if (failCount != 0)
{
if (failCount >= 10)
{
throw new HandledException(string.Format("{0} or more FST errors found", failCount.ToString()));
}
}
if (filesMoved != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be repositioned when rebuilding this image", filesMoved.ToString(), filesMoved == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (nkitJunkFiles != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be generated from junk when rebuilding this image", nkitJunkFiles.ToString(), nkitJunkFiles == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (goodFst != null)
{
//is the data output so far correct
if (!bruteForceValidHeader(ctx, goodFst, hdr, bi2bin, appldr, maindol, fst, pc))
{
throw new HandledException(string.Format("Post FST Crc Failed 0x{0}", goodFst.PostFstCrc.ToString("X8")));
}
}
//############################################################################
//# WRITE DISC START
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
long dstPos = hdr.Data.Length;
crc.Snapshot("boot.bin");
target.Write(bi2bin.Data, 0, (int)bi2bin.Size);
dstPos += bi2bin.Size;
crc.Snapshot("bi2.bin");
target.Write(appldr.Data, 0, (int)appldr.Size);
dstPos += appldr.Size;
ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos);
dstPos += Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos;
crc.Snapshot("appldr.bin");
target.Write(maindol.Data, 0, (int)maindol.Size);
dstPos += maindol.Size;
if (goodFst != null)
{
long padding = goodFst.FstOffset - dstPos;
ByteStream.Zeros.Copy(target, padding);
dstPos += padding;
}
crc.Snapshot("main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
dstPos += fst.Size;
bool firstFile = true;
Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>();
crc.Snapshot("fst.bin");
//############################################################################
//# WRITE THE FILESYSTEM
int fidx = -1;
FstFile lastFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = fst.Size
};
FstFile nextFile = fstFiles[++fidx];
FstFile cacheFile = null;
long nullsPos = dstPos + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
//########### FILES
foreach (FstFile f in srcFiles) //read the files and write them out as goodFiles (possible order difference)
{
while ((cacheFile = cache.Keys.FirstOrDefault(a => (a.Length == nextFile.Length || (inStream.IsNkit && a.Length == 0)) && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) //write cache
{
bool isJunk = inStream.IsNkit && cacheFile.Length == 0 && cacheFile.Name == nextFile.Name && cacheFile.Path == nextFile.Path;
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, isJunk);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
if (f.DataOffset - srcPos > 0) //skip src junk
{
inStream.Copy(ByteStream.Zeros, f.DataOffset - srcPos);
srcPos += f.DataOffset - srcPos;
}
if (nextFile != null && (f.Length != nextFile.Length || f.Name != nextFile.Name || f.Path != nextFile.Path)) //cache file (nkit junk files are cached)
{
bool isNkitJunk = inStream.IsNkit && f.Length == 0 && nextFile.Length != 0 && f.Name == nextFile.Name && f.Path == nextFile.Path;
byte[] cacheItem = new byte[isNkitJunk ? 8 : f.Length];
inStream.Read(cacheItem, 0, cacheItem.Length); //read the nkit junk data - details (real length and null counts)
cache.Add(f, cacheItem);
srcPos += cacheItem.Length;
}
else //copy file
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, inStream, lastFile, nextFile, patches, inStream, junkStart, false);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
srcPos += f.Length;
}
}
while (nextFile != null && fidx < fstFiles.Count && (cacheFile = cache.Keys.FirstOrDefault(a => a.Length == nextFile.Length && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null)
{
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, false);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
writeDestGap(nullsPos, target, dstPos, pc.OutputSize - dstPos, true, patches, junkStart, inStream);
crc.Snapshot("files");
}
resultMsg = "MatchFail";
uint finalCrc = crc.FullCrc(true);
if (ctx.Dats.RedumpData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Redump";
}
if (ctx.Dats.CustomData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Custom";
}
pc.ReaderCheckPoint2Complete(crc, false, finalCrc, finalCrc, VerifyIsWrite, hdr.Data, resultMsg);
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "RestoreReaderGc.Read - Read and repair"); //don't let the writer lock
}
}
private long writeGap(ConvertFile file, ref long nullsPos, ref long srcPos, long dstPos, NStream inStream, Stream target, bool firstOrLastFile)
{
if (file.GapLength == 0)
{
if (file.FstFile.Length == 0)
{
nullsPos = dstPos + 0x1c;
}
return(0);
}
MemorySection ms = MemorySection.Read(inStream, 4);
srcPos += 4;
long size = ms.ReadUInt32B(0);
GapType gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
if (size == 0xFFFFFFFC) //for wii only. not a thing for GC
{
srcPos += 4;
inStream.Read(ms.Data, 0, 4);
size = 0xFFFFFFFCL + (long)ms.ReadUInt32B(0); //cater for files > 0xFFFFFFFF
}
long nulls;
long junkFileLen = 0;
//set nullsPos value if zerobyte file without junk
if (gt == GapType.JunkFile)
{
nullsPos = Math.Min(nullsPos - dstPos, 0);
nulls = (size & 0xFC) >> 2;
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
junkFileLen = ms.ReadUInt32B(0);
file.FstFile.Length = junkFileLen;
junkFileLen += junkFileLen % 4 == 0 ? 0 : 4 - (junkFileLen % 4);
ByteStream.Zeros.Copy(target, nulls);
inStream.JunkStream.Position = dstPos + nulls;
inStream.JunkStream.Copy(target, junkFileLen - nulls);
dstPos += junkFileLen;
if (file.GapLength <= 8)
{
return(junkFileLen);
}
else
{
//read gap
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
size = ms.ReadUInt32B(0);
gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
}
}
else if (file.FstFile.Length == 0) //last zero byte file was legit
{
nullsPos = dstPos + 0x1c;
}
long maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (size < maxNulls) //need to test this commented if
{
nulls = size;
}
else
{
nulls = size >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
if (gt == GapType.AllJunk)
{
ByteStream.Zeros.Copy(target, nulls);
inStream.JunkStream.Position = dstPos + nulls;
inStream.JunkStream.Copy(target, size - nulls);
dstPos += size;
}
else if (gt == GapType.AllScrubbed)
{
ByteStream.Zeros.Copy(target, size);
dstPos += size;
}
else
{
long prg = size;
byte btByte = 0x00;
GapBlockType bt = GapBlockType.Junk; //should never be used
while (prg > 0)
{
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
long bytes;
long blk = ms.ReadUInt32B(0);
GapBlockType btType = (GapBlockType)(blk >> 30);
bool btRepeat = btType == GapBlockType.Repeat;
if (!btRepeat)
{
bt = btType;
}
long cnt = 0x3FFFFFFF & blk;
if (bt == GapBlockType.NonJunk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
inStream.Copy(target, bytes);
srcPos += bytes;
}
else if (bt == GapBlockType.ByteFill)
{
if (!btRepeat)
{
btByte = (byte)(0xFF & cnt); //last 8 bits when not repeating are the byte
cnt >>= 8;
}
bytes = Math.Min(cnt * Gap.BlockSize, prg);
Stream bs;
switch (btByte)
{
case 0x00: bs = ByteStream.Zeros; break;
case 0x55: bs = ByteStream.Fives; break;
case 0xff: bs = ByteStream.FFs; break;
default: bs = new ByteStream(btByte); break;
}
bs.Copy(target, bytes);
}
else //if (bt == GapBlockType.Junk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (prg < maxNulls)
{
nulls = bytes;
}
else
{
nulls = bytes >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
ByteStream.Zeros.Copy(target, nulls);
inStream.JunkStream.Position = dstPos + nulls;
inStream.JunkStream.Copy(target, bytes - nulls);
}
prg -= bytes;
dstPos += bytes;
}
}
return(size + junkFileLen);
}
private long writeGap(ref long fileLength, LongRef gapLength, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target, JunkStream junk, bool firstOrLastFile, ScrubManager scrub)
{
if (gapLength.Value == 0)
{
if (fileLength == 0)
{
nullsPos = dstPos + 0x1c;
}
return(0);
}
long srcLen = gapLength.Value; //fix added for (padding between junk files) - Zumba Fitness (Europe) (En,Fr,De,Es,It)
MemorySection ms = MemorySection.Read(inStream, 4);
srcPos += 4;
long size = ms.ReadUInt32B(0);
GapType gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
if (size == 0xFFFFFFFC) //for wii only. not a thing for GC
{
srcPos += 4;
inStream.Read(ms.Data, 0, 4);
size = 0xFFFFFFFCL + (long)ms.ReadUInt32B(0); //cater for files > 0xFFFFFFFF
}
gapLength.Value = size;
scrub.AddGap(fileLength, dstPos, size); //keep track of trailing nulls when restoring scrubbed images
long nulls;
long junkFileLen = 0;
//set nullsPos value if zerobyte file without junk
if (gt == GapType.JunkFile)
{
nullsPos = Math.Min(nullsPos - dstPos, 0);
nulls = (size & 0xFC) >> 2;
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
junkFileLen = ms.ReadUInt32B(0);
fileLength = junkFileLen;
junkFileLen += junkFileLen % 4 == 0 ? 0 : 4 - (junkFileLen % 4);
ByteStream.Zeros.Copy(target, nulls);
junk.Position = dstPos + nulls;
junk.Copy(target, junkFileLen - nulls);
dstPos += junkFileLen;
if (srcLen <= 8)
{
return(junkFileLen);
}
else
{
//read gap
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
size = ms.ReadUInt32B(0);
gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
gapLength.Value = size;
}
}
else if (fileLength == 0) //last zero byte file was legit
{
nullsPos = dstPos + 0x1c;
}
long maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (size < maxNulls) //need to test this commented if
{
nulls = size;
}
else
{
nulls = size >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
nullsPos = dstPos + nulls; //belt and braces
if (gt == GapType.AllJunk)
{
ByteStream.Zeros.Copy(target, nulls);
junk.Position = dstPos + nulls;
junk.Copy(target, size - nulls);
dstPos += size;
}
else if (gt == GapType.AllScrubbed)
{
scrub.Scrub(target, dstPos, size, 0);
dstPos += size;
}
else
{
long prg = size;
byte btByte = 0x00;
GapBlockType bt = GapBlockType.Junk; //should never be used
while (prg > 0)
{
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
long bytes;
long blk = ms.ReadUInt32B(0);
GapBlockType btType = (GapBlockType)(blk >> 30);
bool btRepeat = btType == GapBlockType.Repeat;
if (!btRepeat)
{
bt = btType;
}
long cnt = 0x3FFFFFFF & blk;
if (bt == GapBlockType.NonJunk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
inStream.Copy(target, bytes);
srcPos += bytes;
}
else if (bt == GapBlockType.ByteFill)
{
if (!btRepeat)
{
btByte = (byte)(0xFF & cnt); //last 8 bits when not repeating are the byte
cnt >>= 8;
}
bytes = Math.Min(cnt * Gap.BlockSize, prg);
scrub.Scrub(target, dstPos, bytes, btByte);
}
else //if (bt == GapBlockType.Junk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (prg < maxNulls)
{
nulls = bytes;
}
else
{
nulls = bytes >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
ByteStream.Zeros.Copy(target, nulls);
junk.Position = dstPos + nulls;
junk.Copy(target, bytes - nulls);
}
prg -= bytes;
dstPos += bytes;
}
}
return(gapLength.Value + junkFileLen);
}
private long partitionStreamWrite(LongRef outSize, Stream inStream, Stream target, long size, DatData settingsData, NkitPartitionPatchInfo patchInfo, WiiHashStore hashes, Coordinator pc)
{
DatData data = settingsData;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
long srcPos = hdr.Size;
long outPos = 0;
long imageSize = 0;
try
{
if (hdr.ReadString(0, 4) == "\0\0\0\0")
{
long nullsPos = 0;
long fileLength = -1;
LongRef gapLength = new LongRef()
{
Value = -1
};
target.Write(hdr.Data, 0, (int)hdr.Size);
MemorySection sz = MemorySection.Read(inStream, 4);
srcPos += 4;
outPos += hdr.Size;
imageSize = sz.ReadUInt32B(0) * 4L;
outSize.Value = NStream.HashedLenToData(imageSize);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), outSize.Value); //SET LENGTH FROM HEADER
outPos += writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
imageSize = NStream.HashedLenToData((hdr.ReadUInt32B(0x210) * 4L));
outSize.Value = imageSize;
string junkId = hdr.ReadString(0x214, 4);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), imageSize); //SET LENGTH FROM HEADER
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x424) * 4L) - hdr.Size);
srcPos += hdrToFst.Size;
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) * 4L);
long postFstPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
srcPos += fst.Size;
hashes.WriteFlagsData(imageSize, inStream);
srcPos += hashes.FlagsLength;
patchInfo.PartitionDataHeader = hdr;
patchInfo.Fst = fst;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
target.Write(fst.Data, 0, fst.Data.Length);
hdrToFst = null; //let this be collected if needed
outPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
long nullsPos = outPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, size, hdr, fst, false, -1, out error);
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
outPos += writeGap(cf, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
conFiles[0].GapLength -= hashes.FlagsLength; //fix for a few customs (no gap between the fst and the first file on the source image, but the hash mask makes it look like there is)
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), outPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (outPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset) //skip any padding (not written for wii currently)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)(outPos / 4L));
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(outPos / 4L));
outPos += copyFile(f, ref nullsPos, ref srcPos, outPos, inStream, target);
}
if (outPos < imageSize)
{
long gapLen = writeGap(f, ref nullsPos, ref srcPos, outPos, inStream, target, junk, i == 0 || i == conFiles.Count - 1, patchInfo.ScrubManager);
outPos += gapLen;
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)(ff.Length));
}
}
firstFile = false;
}
}
}
return(srcPos);
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - partitionRead");
}
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
long imageSize = hdr.ReadUInt32B(0x210) * 4L;
string junkId = hdr.ReadString(0x214, 4);
uint updatePartitionCrc = hdr.ReadUInt32B(0x218);
MemorySection updateRemovedFiller = null;
long discOffset = 0;
List <NkitPartitionPatchInfo> patchInfos = new List <NkitPartitionPatchInfo>();
discOffset += hdr.Size;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
NCrc crc = new NCrc();
long dstPos = 0;
long srcPos = hdr.Size;
ScrubManager scrubFiller = new ScrubManager(null);
bool isRecoverable = false;
using (NDisc disc = new NDisc(_log, inStream))
{
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
hdr.Write8(0x60, 0);
hdr.Write8(0x61, 0);
CryptoStream crcStream = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //wrap to calculate crc
crcStream.Write(hdr.Data, 0, hdr.Data.Length); //write the header
pc.ReaderCheckPoint1PreWrite(null, nkitCrc); //size that will be output from this read
dstPos += hdr.Size;
crc.Snapshot("Disc Header");
foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted
{
if (updatePartitionCrc != 0 && srcPos == hdr.Size) //write update partition out
{
updateRemovedFiller = MemorySection.Read(inStream, hdr.Partitions.First().DiscOffset - srcPos);
srcPos += updateRemovedFiller.Size;
WiiPartitionInfo firstPart = WiiDiscHeaderSection.CreatePartitionInfos(updateRemovedFiller, 0)?.FirstOrDefault(a => a.Type != PartitionType.Update);
string updateFileName = RecoveryData.GetUpdatePartition(ctx.Settings, updatePartitionCrc);
if (updateFileName != null)
{
using (FileStream pf = File.OpenRead(updateFileName))
{
pf.Copy(crcStream, pf.Length);
dstPos += pf.Length;
}
}
else
{
string msg = string.Format("!! Update partition *_{0} missing - Adding filler. It may be Recoverable", updatePartitionCrc.ToString("X8"));
isRecoverable = true;
_log?.LogDetail(msg);
//throw pc.SetReaderException(new HandledException("Failed to convert: " + msg));
}
ByteStream.Zeros.Copy(crcStream, firstPart.DiscOffset - dstPos); //fill full gap
dstPos += firstPart.DiscOffset - dstPos;
}
NkitPartitionPatchInfo patchInfo = new NkitPartitionPatchInfo()
{
HashGroups = new Dictionary <long, MemorySection>()
};
patchInfos.Add(patchInfo);
if (part.DiscOffset > srcPos)
{
dstPos += writeFiller(ref srcPos, dstPos, dstPos + 0x1cL, inStream, crcStream, new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize), scrubFiller);
inStream.Copy(ByteStream.Zeros, part.DiscOffset - srcPos); //padded to 0x8000
srcPos += part.DiscOffset - srcPos;
}
part.DiscOffset = dstPos; //restore the original position
patchInfo.DiscOffset = dstPos;
patchInfo.PartitionHeader = MemorySection.Read(inStream, 0x20000);
srcPos += patchInfo.PartitionHeader.Size;
long size = patchInfo.PartitionHeader.ReadUInt32B(0x2bc) * 4L;
LongRef origSize = new LongRef()
{
Value = 0
};
WiiPartitionGroupSection wp = null;
WiiPartitionHeaderSection wh = new WiiPartitionHeaderSection(hdr, null, part.DiscOffset, patchInfo.PartitionHeader.Data, patchInfo.PartitionHeader.Data.Length);
MemorySection ph = new MemorySection(new byte[0x8000 * 64]);
long remaining = long.MaxValue;//set after first block read
int groupIndex = 0;
WiiHashStore hashes = new WiiHashStore();
patchInfo.ScrubManager = wh.ScrubManager;
bool patchBlock = false;
StreamCircularBuffer prtStream = null;
try
{
using (prtStream = new StreamCircularBuffer(0, null, null, output => srcPos += partitionStreamWrite(origSize, inStream, output, size, ctx.Dats, patchInfo, hashes, pc)))
{
int gs = 0;
int ge = 0;
int i = 0;
MemoryStream patchBlocks = null;
while (remaining > 0)
{
int blocks = (int)Math.Min(64L, remaining / 0x7c00);
for (int b = 0; b < blocks; b++)
{
prtStream.Read(ph.Data, (b * 0x8000) + 0x400, 0x7c00); //load aligned with no hashes
if (remaining == long.MaxValue) //first loop
{
remaining = origSize.Value;
if (ph.ReadString(0x400 + 0, 4) == "\0\0\0\0")
{
gs = -1;
ge = -1;
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, (uint)(NStream.DataToHashedLen(origSize.Value) / 4)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
}
else
{
gs = (int)((ph.ReadUInt32B(0x400 + 0x424) * 4L) / (0x7c00L * 64));
ge = (int)(((ph.ReadUInt32B(0x400 + 0x424) * 4L) + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) / (0x7c00L * 64));
if ((int)((part.DiscOffset + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) % (0x7c00L * 64)) == 0)
{
ge--; //don't load the next group if the data will end on the last byte
}
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, ph.ReadUInt32B(0x400 + 0x210)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
ph.WriteUInt32B(0x400 + 0x200, 0);
ph.WriteUInt32B(0x400 + 0x204, 0);
ph.WriteUInt32B(0x400 + 0x208, 0);
ph.WriteUInt32B(0x400 + 0x20C, 0);
ph.WriteUInt32B(0x400 + 0x210, 0);
ph.WriteUInt32B(0x400 + 0x214, 0);
ph.WriteUInt32B(0x400 + 0x218, 0);
}
wp = new WiiPartitionGroupSection(hdr, wh, ph.Data, part.DiscOffset, blocks * 0x8000, false);
wh.Initialise(wp, origSize.Value);
}
}
if (blocks < 64)
{
Array.Clear(ph.Data, blocks * 0x8000, ph.Data.Length - (blocks * 0x8000)); //clear remaining blocks
}
wp.Populate(groupIndex, ph.Data, dstPos, blocks * 0x8000);
int scrubbed = 0;
for (int bi = 0; bi < blocks; bi++)
{
wp.MarkBlockDirty(bi);
byte byt;
if (patchInfo.ScrubManager.IsBlockScrubbedScanMode(wp.Offset + (bi * 0x8000), out byt))
{
wp.SetScrubbed(bi, byt);
scrubbed++;
}
}
bool isFstBlocks = i >= gs && i <= ge;
bool reqHashes = hashes.IsPreserved(wp.Offset); //test with 0 partition based offset
repairBlocks(wp, scrubbed, blocks, false, isFstBlocks); //only test if the hashes aren't preserved (only preserved for scrubbed/customs)
if (reqHashes) //store with disc based offset
{
patchInfo.HashGroups.Add(wp.Offset + part.DiscOffset + patchInfo.PartitionHeader.Size, new MemorySection((byte[])wp.Decrypted.Clone())); //fetch the stored hashed that couldn't be recreated
}
groupIndex++;
bool inFstArea = i >= gs && i <= ge;
if (!patchBlock && (gs == i || reqHashes))
{
patchBlocks = new MemoryStream();
crc.Snapshot(lastPartitionId + " Data");
patchBlock = true;
}
else if (patchBlock && !inFstArea && !reqHashes)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
patchBlock = false;
}
#if DECRYPT
outStream.Write(wp.Decrypted, 0, blocks * 0x8000);
if (i >= gs && i <= ge)
{
fstBlocks.Write(wp.Decrypted, 0, blocks * 0x8000);
}
#else
crcStream.Write(wp.Encrypted, 0, blocks * 0x8000);
if (patchBlock)
{
patchBlocks.Write(wp.Encrypted, 0, blocks * 0x8000);
}
#endif
remaining -= (blocks * 0x7c00);
dstPos += (blocks * 0x8000);
i++;
}
if (patchBlock)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
}
}
if (origSize.Value != prtStream.WriterPosition)
{
throw pc.SetReaderException(new HandledException("Partition read did not write the full amount to the circular buffer before completing"));
}
}
catch (Exception ex)
{
if (prtStream?.WriterException != null)
{
throw pc.SetReaderException(prtStream.WriterException, "Failed reading filesystem");
}
throw pc.SetReaderException(ex, "Failed converting the filesystem");; //writer exception
}
srcPos += hashes.ReadPatchData(part.DiscOffset + patchInfo.PartitionHeader.Size, patchInfo.HashGroups, inStream);
//read hash patches
lastPartitionType = part.Type;
}
if (srcPos < inStream.Length)
{
JunkStream partJunk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
dstPos += writeFiller(ref srcPos, dstPos, lastPartitionType == PartitionType.Update ? imageSize : dstPos + 0x1cL, inStream, crcStream, partJunk, scrubFiller);
}
}
crc.Snapshot("End");
if (updatePartitionCrc != 0)
{
hdr.Write((int)WiiDiscHeaderSection.PartitionTableOffset, updateRemovedFiller.Data, 0, (int)WiiDiscHeaderSection.PartitionTableLength); //restore the exact partition table if update was removed
}
else
{
hdr.UpdateOffsets(); //just update the table with the new offsets
}
crc.Crcs[0].PatchData = hdr.Data;
foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
{
NkitPartitionPatchInfo patchInfo = patchInfos.FirstOrDefault(a => ci.Offset >= a.DiscOffset + a.PartitionHeader.Size && ci.Offset < a.DiscOffset + a.PartitionHeader.Size + a.Size);
if (patchInfo != null)
{
patchGroups(patchInfo, hdr, ci.Offset, ci.PatchData);
}
ci.PatchCrc = Crc.Compute(ci.PatchData);
}
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, isRecoverable, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - Read and convert");
}
}
private long writeFile(ref long nullsPos, Stream dest, long dstPos, ref bool firstFile, Stream srcStream, FstFile lastFile, FstFile file, List <JunkRedumpPatch> patches, NStream nstream, long junkStart, bool isJunkFile)
{
//Debug.WriteLine(string.Format(@"{0} : {1} : {2} : {3}/{4}", file.DataOffset.ToString("X8"), (file.DataOffset + file.Length).ToString("X8"), /*(nextFile.DataOffset - lastEnd).ToString("X8"),*/ file.Length.ToString("X8"), file.Path, file.Name));
//file found
long gap = writeDestGap(nullsPos, dest, dstPos, file.DataOffset - dstPos, firstFile, patches, junkStart, nstream);
firstFile = false;
bool missing = false;
if (file.Length == 0)
{
// missing = true;
}
else if (isJunkFile)
{
missing = true;
MemorySection ms = MemorySection.Read(srcStream, 8);
long size = ms.ReadUInt32B(0);
GapType gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
//set nullsPos value if zerobyte file without junk
if (gt == GapType.JunkFile)
{
nullsPos = Math.Min(nullsPos - (dstPos + gap), 0); //reset the nulls
long nulls = (size & 0xFC) >> 2;
long junkFileLen = ms.ReadUInt32B(4);
if (junkFileLen != file.Length)
{
throw new HandledException(string.Format("NKit Junk file restoration length mismatch {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
ByteStream.Zeros.Copy(dest, nulls);
nstream.JunkStream.Position = (dstPos + gap) + nulls;
nstream.JunkStream.Copy(dest, junkFileLen - nulls);
_log?.LogDetail(string.Format("Generated file content with Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
else
{
throw new HandledException(string.Format("NKit Junk file restoration bytes invalid {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
}
else
{
byte[] f = new byte[Math.Min(0x30, file.Length)];
srcStream.Read(f, 0, f.Length); //then read while junk is created
if (lastFile.DataOffset == file.DataOffset && lastFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size
{
nullsPos = dstPos + 0x1CL; //will already be aligned
}
int nulls = (int)(nullsPos - (dstPos + gap));
nstream.JunkStream.Position = file.DataOffset; //async junk gen
int countNulls = 0;
for (int i = 0; i < f.Length && f[i] == 0; i++)
{
countNulls++;
}
if (f.Length > nulls && countNulls < f.Length) //don't test all nulls
{
missing = nstream.JunkStream.Compare(f, 0, f.Length, Math.Max(0, nulls)) == f.Length;
}
if (missing)
{
_log?.LogDetail(string.Format("File content is Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
dest.Write(f, 0, f.Length);
srcStream.Copy(dest, file.Length - f.Length); //copy file
}
if (!missing) //reset the gap when no junk
{
nullsPos = dstPos + gap + file.Length + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
}
return(gap + file.Length);
}
internal bool Initialize(bool readHeader)
{
if (_id != null)
{
return(_headerRead);
}
List <long> IsoDecParts = new List <long>();
_headerRead = readHeader;
try
{
if (readHeader)
{
_currentBlockIndex = -1;
_id = read(4);
string id = Encoding.ASCII.GetString(_id);
if (id == "WBFS")
{
uint hdSecSize = this.readUInt32B(); //n_hd_sec
long pos = (uint)(1 << this.read8()) + 0x100;
_clusterSize = 1 << this.read8();
int clusters = (int)(FullSizeWii5 / _clusterSize) * 2;
//if ((hdSecSize * _clusterSize) / 0x1000L != _stream.Length) //WBFS header check
// Debug.WriteLine("MatchFail");
_stream.Copy(ByteStream.Zeros, pos - 10);
MemorySection ms = MemorySection.Read(_stream, 2 * clusters);
uint pointer;
long maxPointer = 0;
long blank = 0;
long blankTmp = 0;
long fileSize = _stream.Length;
for (int i = 0; i < clusters; i++)
{
if ((pointer = ms.ReadUInt16B(i * 2)) != 0)
{
_imageSize += _clusterSize;
}
if (pointer == 0)
{
blankTmp++;
}
else if (pointer * (long)_clusterSize < fileSize)
{
blank += blankTmp;
blankTmp = 0;
maxPointer = Math.Max(maxPointer, pointer * (long)_clusterSize);
}
_clusterTable.Add(pointer);
}
_streamDataStart = _stream.Position;
_imageSize = _readLength = lenCalc(Math.Max(maxPointer + (blank * _clusterSize), _imageSize));
_isWbfs = _isWii = true; //must set after above code has ran
setDiscHeader(false, null);
}
else if ((id == "WII5" || id == "WII9") && !_isGamecube) //IsoDec :(
{
int sectorSize = id == "WII5" ? 0x1182800 : 0x1FB5000;
string discId = this.readString(4);
byte[] md5 = this.read(16);
int partitions = (int)this.readUInt32L();
for (int i = 0; i < partitions; i++)
{
var part = new //throw away
{
DataOffset = (long)((ulong)this.readUInt32L() << 2),
DataSize = (long)((ulong)this.readUInt32L() << 2),
PartitionOffset = (long)((ulong)this.readUInt32L() << 2),
PartitionEndOffset = (long)((ulong)this.readUInt32L() << 2),
PartitionKey = this.read(16)
};
IsoDecParts.Add(part.PartitionOffset);
}
;
_clusterSize = 0x400; //1k
_clusterTable = new List <uint>();
int c = (sectorSize - (28 + (partitions * 32))) / 4;
MemorySection ms = MemorySection.Read(_stream, 4 * c);
for (int i = 0; i < c; i++)
{
_clusterTable.Add(ms.ReadUInt32L(i * 4));
}
_imageSize = _readLength = (id == "WII5" ? FullSizeWii5 : FullSizeWii9);
_isIsoDec = _isWii = true;
_streamDataStart = _stream.Position;
setDiscHeader(false, IsoDecParts.ToArray());
}
else if (id == "GCML") //GC IsoDec
{
string discId = this.readString(4);
byte[] md5 = this.read(16);
_clusterTable = new List <uint>();
_clusterSize = 0x800; //2k
int c = (0x2B8800 - 24) / 4;
MemorySection ms = MemorySection.Read(_stream, 4 * c);
for (int i = 0; i < c; i++)
{
_clusterTable.Add(ms.ReadUInt32L(i * 4));
}
_streamDataStart = _stream.Position;
_IsoDecMultiply = 0x1L; //none
_imageSize = _readLength = FullSizeGameCube;
_isIsoDec = _isGamecube = true;
setDiscHeader(false, null);
}
else if (BitConverter.ToString(_id) == "01-C0-0B-B1") //GCZ - C001B10B as big endian
{
MemorySection ms = MemorySection.Read(_stream, 0x20 - 4);
long compSize = (long)ms.ReadUInt64L(0x8 - 4);
_imageSize = _readLength = (long)ms.ReadUInt64L(0x10 - 4);
_clusterSize = (int)ms.ReadUInt32L(0x18 - 4);
uint blocks = ms.ReadUInt32L(0x1C - 4);
_currentBlock = new byte[_clusterSize];
MemorySection pnt = MemorySection.Read(_stream, blocks * 8);
MemorySection hsh = MemorySection.Read(_stream, blocks * 4);
ulong dataOffset = (ulong)(pnt.Size + hsh.Size + 0x20);
_clusterTable = new List <uint>();
_clusterTableCompressed = new List <uint>();
for (int i = 0; i < blocks; i++)
{
ulong offset = pnt.ReadUInt64L(i * 8) + dataOffset; //offset in raw file
bool compressed = (offset & 0x8000000000000000) == 0;
offset &= ~0x8000000000000000;
_clusterTable.Add((uint)offset); //divide by 4 for wii
_clusterTableCompressed.Add((uint)(compressed ? 1 : 0));
}
_isGcz = true;
MemorySection xx = MemorySection.Read(this, 4);
_id = xx.Read(0, 4);
setDiscHeader(true, null);
}
else
{
_streamDataStart = 0;
_isIso = true;
_imageSize = _readLength = _stream.Length;
setDiscHeader(true, null); //true sets length too
_clusterSize = _isWii ? 0x400 : 0x800;
}
this.ResetJunk();
return(true);
}
else
{
_imageSize = _readLength = _stream.Length; //partial streams not disc
this.Position = 0;
_streamDataStart = 0;
}
return(false);
}
catch (Exception ex)
{
throw new HandledException(ex, "NStream.Initialize - Read Header");
}
}