internal void RemoveUpdatePartition(long baseAddress)
{
if (this.Partitions.Length == 0 || this.Partitions[0].Type != PartitionType.Update)
{
return;
}
_partitions.RemoveAt(0);
Array.Clear(this.Data, 0x60, 2);
Array.Clear(this.Data, _PartitionTableOffset, _PartitionTableLength);
//write the partition info
MemorySection hdrStream = new MemorySection(this.Data);
WiiPartitionInfo firstNonUpdate = _partitions.FirstOrDefault(a => a.Type != PartitionType.Update);
foreach (IGrouping <int, WiiPartitionInfo> grp in _partitions.GroupBy(a => a.Table))
{
int offset = (int)(_PartitionTableOffset + 0x20 + (grp.Key * 0x20L));
hdrStream.WriteUInt32B((int)(_PartitionTableOffset + (grp.Key * 0x8L)), (uint)grp.Count());
hdrStream.WriteUInt32B((int)(_PartitionTableOffset + (grp.Key * 0x8L) + 4), (uint)(offset / 4));
offset -= 4; //adjust for the first calc
foreach (WiiPartitionInfo part in grp)
{
hdrStream.WriteUInt32B(offset += 4, (uint)(part.DiscOffset / 4L));
part.TableOffset = offset;
hdrStream.WriteUInt32B(offset += 4, (uint)(part.Type));
}
}
}
public void SaveMainDol(string outPath)
{
MemorySection hdr = this.NStream.DiscHeader;
this.NStream.Copy(ByteStream.Zeros, hdr.ReadUInt32B(0x420) - 0x440);
//dol info https://github.com/jchv/gcmtools/blob/master/gcm/dol.hpp
byte[] dolLen = new byte[4];
this.NStream.Read(dolLen, 0, 4);
MemorySection mx = new MemorySection(dolLen);
byte[] dolHdr = new byte[mx.ReadUInt32B(0)];
dolLen.CopyTo(dolHdr, 0);
this.NStream.Read(dolHdr, 4, dolHdr.Length - 4);
MemorySection maindol = new MemorySection(dolHdr);
uint maindolSize = 0;
for (int i = 0; i < 18; i++)
{
if (maindol.ReadUInt32B(0x0 + (i * 4)) != 0) //7 text offsets, 11 data offsets
{
maindolSize = Math.Max(maindolSize, maindol.ReadUInt32B(0x0 + (i * 4)) + maindol.ReadUInt32B(0x90 + (i * 4)));
}
}
byte[] dol = new byte[maindol.Size + maindolSize];
dolHdr.CopyTo(dol, 0);
this.NStream.Read(dol, dolHdr.Length, dol.Length - dolHdr.Length);
string dolName = Path.Combine(outPath, string.Format("main[{0}][{1}][{2}].dol", this.NStream.Id8, hdr.ReadUInt32B(0x420).ToString("X8"), Crc.Compute(dol).ToString("X8")));
if (!File.Exists(dolName))
{
File.WriteAllBytes(dolName, dol);
}
}
internal WiiDiscHeaderSection(MemorySection header) : base(header.Data)
{
List <WiiPartitionInfo> partitions = new List <WiiPartitionInfo>();
this.Title = this.ReadStringToNull(0x20, 0x60);
_partitions = CreatePartitionInfos(header, _PartitionTableOffset).ToList();
this.HasUpdatePartition = _partitions[0].Type == PartitionType.Update;
}
public static FileSystem Parse(Stream fstData, long fstOffset, long length, string id, bool isGc)
{
MemorySection ms = MemorySection.Read(fstData, length);
FstFile ff = new FstFile(null)
{
Name = "fst.bin", DataOffset = fstOffset, Offset = NStream.DataToOffset(fstOffset, !isGc), IsNonFstFile = true, Length = (int)fstData.Length
};
return(Parse(ms, ff, id, isGc));
}
public HeaderBruteForcer(uint[] updateCrcs, SortedList <uint, bool> checkCrcs, Tuple <byte[], int[]>[] regionData, params byte[][] headers)
{
_updateCrcs = updateCrcs;
_checkCrcs = checkCrcs;
//DateTime dt = DateTime.Now;
_hdrs = headers.Select(a => new headerCrc()
{
Crc = Crc.Compute(a), CrcPreRegion = Crc.Compute(a, 0, 0x4e000), CrcRegion = Crc.Compute(a, 0x4e000, 0x20), CrcPostRegion = Crc.Compute(a, 0x4e020, 0x1fe0), Data = a
}).ToList();
MemorySection ms = new MemorySection(headers[0]);
_origRegion = (int)ms.ReadUInt32B(0x4e000);
_origRegionData = ms.Read(0x4e010, 0x10);
_origHeaderCrcs = _hdrs.Select(a => a.Crc).ToArray(); //original header Crcs
for (int h = _hdrs.Count - 1; h >= 1; h--)
{
if (_hdrs[h].Crc == _hdrs[h - 1].Crc)
{
_hdrs.RemoveAt(h); //remove duplicate headers
}
}
_regionData = new List <regionCrc>();
for (int h = 0; h < _hdrs.Count; h++)
{
for (int i = 0; i < regionData.Length; i++)
{
for (int r = 0; r < regionData[i].Item2.Length; r++)
{
_regionData.Add(new regionCrc()
{
Crc = 0, Header = _hdrs[h], RegionData = regionData[i].Item1, Region = (int)regionData[i].Item2[r]
});
}
}
}
//parallel precalculate the crcs for the matching
Parallel.ForEach(_regionData, rgn =>
{
MemorySection rg = new MemorySection(new byte[0x20]);
rg.WriteUInt32B(0, (uint)rgn.Region);
rg.Write(0x10, rgn.RegionData);
rgn.Crc = ~Crc.Combine(Crc.Combine(~rgn.Header.CrcPreRegion, ~Crc.Compute(rg.Data), 0x20), ~rgn.Header.CrcPostRegion, 0x1fe0);
});
//Trace.WriteLine(string.Format("Crc Headers: {0} - Took: {1}", _regionData.Count.ToString(), (DateTime.Now - dt).TotalMilliseconds.ToString()));
}
public void UpdateOffsets()
{
MemorySection hdrStream = new MemorySection(this.Data);
foreach (IGrouping <int, WiiPartitionInfo> grp in _partitions.GroupBy(a => a.Table))
{
foreach (WiiPartitionInfo part in grp)
{
hdrStream.WriteUInt32B((int)part.TableOffset, (uint)(part.DiscOffset / 4L));
}
}
}
public void Populate()
{
if (!string.IsNullOrEmpty(this.Filename) && File.Exists(this.Filename))
{
MemorySection ms = new MemorySection(File.ReadAllBytes(this.Filename));
this.MainDolOffset = ms.ReadUInt32B(0x00);
this.FstOffset = ms.ReadUInt32B(0x04);
this.MaxFst = ms.ReadUInt32B(0x08);
this.Region = (Region)ms.ReadUInt32B(0x0C);
this.Title = ms.Read(0x10, 0x50 - 0x10);
this.FstData = ms.Read(0x50, (int)ms.Size - 0x50);
}
}
private BruteForceCrcResult results(headerCrc matchHdr, regionCrc matchRgn, uint updateCrc, uint matchCrc, bool isRedump)
{
if (matchHdr == null && matchRgn == null)
{
return new BruteForceCrcResult()
{
Region = _origRegion
}
}
;
//create the header
byte[] header = (matchHdr ?? matchRgn.Header).Data;
bool regionDataChanged = false;
if (matchRgn != null)
{
MemorySection ms = new MemorySection(header);
regionDataChanged = !matchRgn.Region.Equals(_origRegionData);
ms.WriteUInt32B(0x4e000, (uint)matchRgn.Region);
ms.Write(0x4e010, matchRgn.RegionData);
}
uint headerCrc = Crc.Compute(header);
if (headerCrc == _origHeaderCrc)
{
header = null;
}
return(new BruteForceCrcResult()
{
MatchedCrc = matchCrc,
MatchedCrcIsRedump = isRedump,
HeaderChanged = header != null && _origHeaderCrc != headerCrc,
Header = header,
HeaderCrc = Crc.Compute(header),
UpdateChanged = updateCrc != 0 && _origUpdateCrc != updateCrc,
UpdateCrc = updateCrc,
Region = matchRgn?.Region ?? _origRegion,
OriginalRegion = _origRegion,
RegionData = matchRgn?.RegionData,
RegionChanged = matchRgn != null && (regionDataChanged || matchRgn.Region != _origRegion)
});
}
}
public ExtractResult ExtractBasicInfo()
{
Region region;
if (this.IsGameCube)
{
MemorySection bi2bin = MemorySection.Read(this.NStream, 0x2000);
region = (Region)bi2bin.ReadUInt32B(0x18);
}
else
{
region = (Region)this.Header.ReadUInt32B(0x4e000);
}
return(createExtractResult(region, null));
}
internal static FileSystem Parse(MemorySection ms, FstFile fst, string id, bool isGc)
{
FstFolder fld = new FstFolder(null);
long nFiles = ms.ReadUInt32B(0x8);
if (12 * nFiles > ms.Size)
{
return(null);
}
if (fst != null)
{
fld.Files.Add(fst);
}
recurseFst(ms, fld, 12 * nFiles, 0, id, isGc);
return(new FileSystem(fld));
}
public void UpdateRepair()
{
try
{
_partitions.Sort((a, b) => offsetSortFix(a) < offsetSortFix(b) ? -1 : (offsetSortFix(a) > offsetSortFix(b) ? 1 : 0));
Array.Clear(this.Data, 0x60, 2);
Array.Clear(this.Data, _PartitionTableOffset, _PartitionTableLength);
//write the partition info
MemorySection hdrStream = new MemorySection(this.Data);
long partOffsetFix = 0;
WiiPartitionInfo firstNonUpdate = _partitions.FirstOrDefault(a => a.Type != PartitionType.Update);
if (firstNonUpdate != null && firstNonUpdate.DiscOffset < 0xF800000)
{
partOffsetFix = 0xF800000 - firstNonUpdate.DiscOffset;
foreach (WiiPartitionInfo pi in _partitions.Where(a => a.Type != PartitionType.Update))
{
pi.DiscOffset += partOffsetFix;
}
}
foreach (IGrouping <int, WiiPartitionInfo> grp in _partitions.GroupBy(a => a.Table))
{
int offset = (int)(0x40020 + (grp.Key * 0x20L));
hdrStream.WriteUInt32B((int)(_PartitionTableOffset + (grp.Key * 0x8L)), (uint)grp.Count());
hdrStream.WriteUInt32B((int)(_PartitionTableOffset + (grp.Key * 0x8L) + 4), (uint)(offset / 4));
offset -= 4; //adjust for the first calc
foreach (WiiPartitionInfo part in grp)
{
hdrStream.WriteUInt32B(offset += 4, (uint)(part.DiscOffset / 4L));
part.TableOffset = offset;
hdrStream.WriteUInt32B(offset += 4, (uint)(part.Type));
}
}
}
catch (Exception ex)
{
throw new HandledException(ex, "WiiDiscHeaderSection.Update");
}
}
private ExtractResult ExtractFilesWiiNkit(Func <ExtractedFile, bool> filter, Action <Stream, ExtractedFile> extract)
{
WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)Header;
ExtractedFile exFile = new ExtractedFile(IsGameCube ? DiscType.GameCube : DiscType.Wii,
NStream.Id8, null, 0, hdr.Size, "", "hdr.bin", ExtractedFileType.WiiDiscItem);
long srcPos = hdr.Size;
if (filter(exFile))
{
using (MemoryStream ms = new MemoryStream(hdr.Data))
{
extract(ms, exFile);
}
}
foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted
{
if (NStream.Position < part.DiscOffset)
{
NStream.Copy(Stream.Null, part.DiscOffset - NStream.Position);
}
long prtPos = NStream.Position;
MemorySection prtHdr = MemorySection.Read(NStream, 0x20000);
MemorySection prtDataHdr = MemorySection.Read(NStream, 0x440);
string prtId = prtDataHdr.ReadString(0, 4);
if (prtId != "\0\0\0\0")
{
srcPos += prtHdr.Size + prtDataHdr.Size;
exFile = new ExtractedFile(IsGameCube ? DiscType.GameCube : DiscType.Wii,
NStream.Id8, null, prtPos, prtHdr.Size, "", prtId + "hdr.bin", ExtractedFileType.WiiDiscItem);
if (filter(exFile))
{
using (MemoryStream ms = new MemoryStream(prtHdr.Data))
{
extract(ms, exFile);
}
}
extractFiles(prtId, prtDataHdr, NStream, filter, extract);
}
}
return(createExtractResult((Region)Header.ReadUInt32B(0x4e000), null));
}
private void patchGroups(NkitPartitionPatchInfo patchInfo, WiiDiscHeaderSection discHeader, long crcPatchOffset, byte[] crcPatchData)
{
long partDataOffset = patchInfo.DiscOffset + patchInfo.PartitionHeader.Size;
int groupIdx = (int)((crcPatchOffset - partDataOffset) / WiiPartitionSection.GroupSize);
byte[] data = new byte[WiiPartitionSection.GroupSize];
WiiPartitionHeaderSection wh = new WiiPartitionHeaderSection(discHeader, null, partDataOffset, patchInfo.PartitionHeader.Data, patchInfo.PartitionHeader.Size)
{
ScrubManager = patchInfo.ScrubManager
};
wh.Initialise(true, patchInfo.PartitionDataHeader.ReadString(0, 4));
#if DECRYPT
WiiPartitionGroupSection wp = new WiiPartitionGroupSection(discHeader, wh, ph.Data, , ph.Data.Length, false);
#else
WiiPartitionGroupSection wp = new WiiPartitionGroupSection(discHeader, wh, data, partDataOffset, Math.Min(WiiPartitionSection.GroupSize, crcPatchData.Length), true);
#endif
if (patchInfo.Fst != null)
{
using (MemoryStream fstStream = new MemoryStream(patchInfo.Fst.Data))
{
fstPatch(patchInfo, wp, crcPatchOffset, crcPatchData, fstStream);
}
}
MemorySection d = new MemorySection(crcPatchData);
for (long i = 0; i < crcPatchData.Length; i += WiiPartitionSection.GroupSize)
{
Array.Copy(d.Data, (int)i, data, 0, Math.Min(WiiPartitionSection.GroupSize, d.Size - i));
wp.Populate(groupIdx++, data, crcPatchOffset + i, Math.Min(WiiPartitionSection.GroupSize, d.Size - i));
wp.ForceHashes(null);
if (patchInfo.HashGroups.ContainsKey(wp.DiscOffset))
{
hashPatchGroup(patchInfo, wp);
d.Write((int)i, wp.Encrypted, (int)wp.Size);
}
}
}
private static uint recurseFst(MemorySection ms, FstFolder folder, long names, uint i, string id, bool isGc)
{
uint j;
uint hdr = ms.ReadUInt32B((int)(12 * i));
long name = names + hdr & 0x00ffffffL;
int type = (int)(hdr >> 24);
string nm = ms.ReadStringToNull((int)name, Encoding.GetEncoding("shift-jis"));
uint size = ms.ReadUInt32B((int)(12 * i + 8));
if (type == 1)
{
FstFolder f = i == 0 ? folder : new FstFolder(folder)
{
Name = nm
};
if (i != 0)
{
folder.Folders.Add(f);
}
for (j = i + 1; j < size;)
{
j = recurseFst(ms, f, names, j, id, isGc);
}
return(size);
}
else
{
int pos = (int)(12 * i + 4);
long doff = ms.ReadUInt32B(pos) * (isGc ? 1L : 4L); //offset in data
size = ms.ReadUInt32B((int)(12 * i + 8));
long off = NStream.DataToOffset(doff, !isGc); //offset in raw partition
folder.Files.Add(new FstFile(folder)
{
DataOffset = doff, Offset = off, Length = size, Name = nm, PartitionId = id, OffsetInFstFile = pos
});
return(i + 1);
}
}
private void setDiscHeader(bool autodetect, long[] isoDecParts)
{
if (autodetect)
{
//4 bytes have already been read
//read the next part of the header
byte[] x = new byte[0x20];
_id.CopyTo(x, 0);
this.Read(x, 4, 0x20 - 4);
_isWii = x[0x18] == 0x5d && x[0x19] == 0x1c && x[0x1a] == 0x9e && x[0x1b] == 0xa3;
_isGamecube = x[0x1c] == 0xc2 && x[0x1d] == 0x33 && x[0x1e] == 0x9f && x[0x1f] == 0x3d;
if (!_isWii && !_isGamecube)
{
_isGamecube = true; //v1.1 BugFix for Dodger Demo_shrunk.gcm
}
byte[] h = new byte[_isWii ? _HeaderSizeWii : _HeaderSizeGc];
x.CopyTo(h, 0);
this.Read(h, x.Length, h.Length - x.Length);
this.DiscHeader = new MemorySection(h);
}
else
{
this.DiscHeader = MemorySection.Read(this, _isWii ? _HeaderSizeWii : _HeaderSizeGc);
}
if (this.IsWii)
{
this.DiscHeader = new WiiDiscHeaderSection(this.DiscHeader);
((WiiDiscHeaderSection)this.DiscHeader).IsoDecPartitions = isoDecParts;
}
_isNkit = this.DiscHeader.ReadString(0x200, 4) == "NKIT";
_isNkitUpdateRemoved = this.DiscHeader.ReadUInt32B(0x218) != 0; //update not 0means the update partition has been removed
if (_isNkit)
{
_imageSize = this.DiscHeader.ReadUInt32B(0x210) * (_isWii ? 4L : 1L);
}
_position = this.DiscHeader.Size;
}
internal void Initialise(WiiPartitionGroupSection firstSection)
{
_firstSection = firstSection;
IsEncrypted = firstSection.IsEncrypted;
MemorySection ms = new MemorySection(firstSection.Decrypted);
this.Id = ms.ReadString(0x400, 4);
this.DiscNo = (int)ms.Read8(0x406);
if (this.Stream != null)
{
this.Stream.ChangeJunk(this.DiscOffset + _dataOffset, this.Id, this.DiscNo, PartitionDataSize);
}
if (this.Id != "\0\0\0\0")
{
_dolOffset = ms.ReadUInt32B(0x820) * 4L; //+400 to skip hashes
FstOffset = ms.ReadUInt32B(0x824) * 4L;
FstSize = ms.ReadUInt32B(0x828) * 4L;
}
}
internal static IEnumerable <WiiPartitionInfo> CreatePartitionInfos(MemorySection section, int offset)
{
for (int tableIdx = 0; tableIdx < 4; tableIdx++) //up to 4 partitions on the disk
{
uint c = section.ReadUInt32B(offset + (tableIdx * 8)); //count of partitions for tableIdx
//_log.Message(string.Format("Table {0} - Partitions {1}", tableIdx.ToString(), c.ToString("X8")), 2);
if (c == 0)
{
continue;
}
int tableOffset = (int)section.ReadUInt32B(offset + (tableIdx * 8) + 4) * 4; //first partition entry for tableIdx
int adjustReadOffset = offset + (tableOffset - _PartitionTableOffset);
for (int i = 0; i < c; i++)
{
long partitionOffset = section.ReadUInt32B(adjustReadOffset + (i * 8)) * 4L;
PartitionType partitionType = (PartitionType)section.ReadUInt32B(adjustReadOffset + (i * 8) + 4);
//_log.Message(string.Format(" PartitionOffset Offset {0} - Type {1}", partitionOffset.ToString("X8"), partitionType.ToString()), 2);
yield return(new WiiPartitionInfo(partitionType, partitionOffset, tableIdx, tableOffset + (i * 8)));
//_log.Message(string.Format(" ID {0}", partitions.Last().ReadStream.Id), 2);
}
}
}
public ExtractResult ExtractRecoveryFilesGc()
{
MemorySection hdr = NStream.DiscHeader;
MemorySection bi2bin;
List <ExtractRecoveryResult> result = new List <ExtractRecoveryResult>();
NCrc crc = new NCrc();
uint appLdrCrc = 0;
MemorySection fst = null;
uint fstCrc = 0;
string fn;
string tmpFullName;
Log?.Log(string.Format("Processing: {0}", SourceFileName));
int storeType;
using (CryptoStream bw = new CryptoStream(ByteStream.Zeros, crc, CryptoStreamMode.Write))
{
bw.Write(hdr.Data, 0, (int)hdr.Size);
bi2bin = MemorySection.Read(NStream, 0x2000);
bw.Write(bi2bin.Data, 0, (int)bi2bin.Size);
MemorySection aplHdr = MemorySection.Read(NStream, 0x20);
byte[] appLdr = new byte[0x20 + aplHdr.ReadUInt32B(0x14) + aplHdr.ReadUInt32B(0x18)];
Array.Copy(aplHdr.Data, appLdr, (int)aplHdr.Size);
NStream.Read(appLdr, (int)aplHdr.Size, appLdr.Length - (int)aplHdr.Size);
appLdrCrc = Crc.Compute(appLdr);
fn = string.Format("appldr[{0}][{1}].bin", aplHdr.ReadString(0, 10).Replace("/", ""), appLdrCrc.ToString("X8"));
Log?.Log(string.Format(" 1 of 2 - Extracting appldr.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + appLdrCrc.ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, appLdr);
if ((storeType = storeRecoveryFile(this.Settings.RedumpAppldrCrcs.Contains(appLdrCrc), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
bw.Write(appLdr, 0, appLdr.Length); //add to fullcrc
NStream.Copy(bw, hdr.ReadUInt32B(0x424) - (0x2440 + appLdr.Length));
byte[] fstData = new byte[(int)((4 * 4) + (0x60 - 0x20) + hdr.ReadUInt32B(0x428))];
fst = new MemorySection(fstData); //info: dolAddr, fstAddr, maxfst, region, title, fst
NStream.Read(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
fst.WriteUInt32B(0x00, hdr.ReadUInt32B(0x420)); //dol
fst.WriteUInt32B(0x04, hdr.ReadUInt32B(0x424)); //fstAddr
fst.WriteUInt32B(0x08, hdr.ReadUInt32B(0x42C)); //maxfst
fst.WriteUInt32B(0x0c, bi2bin.ReadUInt32B(0x18)); //region
fst.Write(0x10, hdr.Read(0x20, 0x60 - 0x20)); //title
fstCrc = Crc.Compute(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428)); //fst
bw.Write(fstData, (4 * 4) + (0x60 - 0x20), (int)hdr.ReadUInt32B(0x428));
crc.Snapshot("crc");
}
//fst checksums are postfst
fn = string.Format("fst[{0}][{1}][{2}][{3}].bin", NStream.Id8, appLdrCrc.ToString("X8"), fstCrc.ToString("X8"), crc.FullCrc().ToString("X8"));
Log?.Log(string.Format(" 2 of 2 - Extracting fst.bin Recovery File: {0}", fn));
tmpFullName = Path.Combine(Settings.OtherRecoveryFilesPath, fn + "_TEMP") + crc.FullCrc().ToString("X8");
if (File.Exists(tmpFullName))
{
File.Delete(tmpFullName);
}
File.WriteAllBytes(tmpFullName, fst.Data);
if ((storeType = storeRecoveryFile(this.Settings.RedumpFstCrcs.Contains(crc.FullCrc()), tmpFullName, fn, Settings.RecoveryFilesPath, Settings.OtherRecoveryFilesPath, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fn, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = true
});
}
return(createExtractResult((Region)bi2bin.Read8(0x18), result.ToArray()));
}
public virtual OutputResults Process(Context ctx, NStream input, Stream output)
{
OutputResults results = new OutputResults()
{
Conversion = ctx.ConversionName, VerifyOutputResult = VerifyResult.Unverified, ValidateReadResult = VerifyResult.Unverified
};
try
{
StreamCircularBuffer fs = null;
_timer = new Timer
{
Interval = 250
};
_timer.Elapsed += (s, e) =>
{
if (_timerRunning)
{
return; //keep processing
}
try
{
_timerRunning = true;
_log.ProcessingProgress(((IProgress)fs)?.Value ?? 0);
}
catch { }
finally
{
_timerRunning = false;
}
};
_timer.Enabled = true;
long size;
switch (_sizeMode)
{
case ProcessorSizeMode.Source:
size = input.SourceSize;
break;
case ProcessorSizeMode.Stream:
size = input.Length;
break;
case ProcessorSizeMode.Image:
size = input.ImageSize;
break;
case ProcessorSizeMode.Recover:
default:
size = input.RecoverySize;
break;
}
Coordinator pc = new Coordinator(ctx.ValidationCrc, Reader, Writer, size);
pc.Started += (s, e) =>
{
_timer.Enabled = true;
results.AliasJunkId = e.AliasJunkId;
};
pc.Completed += (s, e) =>
{
_timer.Enabled = false;
if (Writer is HashWriter)
{
results.OutputMd5 = e.Md5;
results.OutputSha1 = e.Sha1;
}
else
{
MemorySection hdr = new MemorySection(e.Header ?? input.DiscHeader.Data);
results.OutputTitle = hdr.ReadStringToNull(0x20, 0x60);
results.OutputDiscNo = hdr.Read8(6);
results.OutputDiscVersion = hdr.Read8(7);
results.OutputId8 = string.Concat(hdr.ReadString(0, 6), results.OutputDiscNo.ToString("X2"), results.OutputDiscVersion.ToString("X2"));
results.ProcessorMessage = e.ResultMessage;
results.OutputCrc = e.PatchedCrc;
results.IsRecoverable = e.IsRecoverable;
if (e.ValidationCrc != 0)
{
results.ValidationCrc = e.ValidationCrc;
results.ValidateReadResult = e.ValidationCrc == e.PatchedCrc ? VerifyResult.VerifySuccess : VerifyResult.VerifyFailed;
}
if (!(Writer is VerifyWriter))
{
results.OutputSize = e.OutputSize; //never store the verify size
}
else
{
results.VerifyCrc = e.VerifyCrc;
if (e.VerifyIsWrite) //e.ValidationCrc can be set from a previous process run
{
results.VerifyOutputResult = results.ValidationCrc == results.VerifyCrc ? VerifyResult.VerifySuccess : VerifyResult.VerifyFailed;
}
}
}
bool l9 = pc.Patches.Crcs.Any(a => a.Offset > 0xFFFFFFFFL || a.Length > 0xFFFFFFFFL);
if (pc.ReaderCrcs != null)
{
foreach (CrcItem c in pc.ReaderCrcs.Crcs)
{
_log.LogDebug(string.Format("R-CRC {0} Before:{1} After:{2} L:{3} {4}", c.Offset.ToString(l9 ? "X9" : "X8"), c.Value.ToString("X8"), c.PatchCrc == 0 ? " " : c.PatchCrc.ToString("X8"), c.Length.ToString(l9 ? "X9" : "X8"), SourceFiles.CleanseFileName(c.Name)));
}
_log.LogDebug(string.Format("ReadCRC {0}Before:{1} After:{2}", l9 ? " " : "", pc.ReaderCrcs.FullCrc(false).ToString("X8"), pc.ReaderCrcs.FullCrc(true).ToString("X8")));
}
if (pc.WriterCrcs != null)
{
foreach (CrcItem c in pc.WriterCrcs.Crcs)
{
_log.LogDebug(string.Format("W-CRC {0} Before:{1} After:{2} L:{3} {4}", c.Offset.ToString(l9 ? "X9" : "X8"), c.Value.ToString("X8"), c.PatchCrc == 0 ? " " : c.PatchCrc.ToString("X8"), c.Length.ToString(l9 ? "X9" : "X8"), SourceFiles.CleanseFileName(c.Name)));
}
_log.LogDebug(string.Format("WriteCRC {0}Before:{1} After:{2}", l9 ? " " : "", pc.WriterCrcs.FullCrc(false).ToString("X8"), pc.WriterCrcs.FullCrc(true).ToString("X8")));
}
_log.ProcessingComplete(results.OutputSize, results.ProcessorMessage, true);
};
try
{
_log.ProcessingStart(input.SourceSize, Title);
using (fs = new StreamCircularBuffer(size, input, null, s => Reader.Read(ctx, input, s, pc))) //read in stream and write to circular buffer
{
Writer.Write(ctx, fs, output, pc);
}
}
catch
{
if (pc.Exception != null)
{
throw pc.Exception;
}
if (fs.WriterException != null)
{
throw fs.WriterException;
}
throw; //writer exception
}
foreach (CrcItem crc in pc.Patches.Crcs.Where(a => a.PatchData != null || a.PatchFile != null))
{
output.Seek(crc.Offset, SeekOrigin.Begin);
if (crc.PatchFile == null)
{
output.Write(crc.PatchData, 0, (int)Math.Min(crc.Length, crc.PatchData.Length)); //PatchData might be larger
}
else
{
using (FileStream pf = File.OpenRead(crc.PatchFile))
{
pf.Copy(output, pf.Length);
ByteStream.Zeros.Copy(output, crc.Length - pf.Length);
}
}
}
}
catch (Exception ex)
{
if (_timer != null)
{
_timer.Enabled = false;
}
try
{
_log.ProcessingComplete(results.OutputSize, results.ProcessorMessage, false); // force any log lines to be output - handy for diagnosis
}
catch { }
throw new HandledException(ex, "Failed processing {0} -> {1}", Reader?.GetType()?.Name ?? "<null>", Writer?.GetType()?.Name ?? "<null>");
}
finally
{
if (_timer != null)
{
_timer.Enabled = false;
_timer = null;
}
}
return(results);
}
internal static List <ConvertFile> GetConvertFstFiles(Stream inStream, long size, MemorySection hdr, MemorySection fst, bool isGc, long fstFileAlignment, out string error)
{
string[] align = new string[] { ".tgc" }; //, ".dol", ".thp", ".adp", ".poo", ".pcm", ".pcm16", ".son", ".act", ".bin", ".gpl", ".tpl", ".bmp", ".sam", ".sdi", ".tcs", ".txt", ".skn", ".stp", ".anm" };
error = null;
List <ConvertFile> conFiles = new List <ConvertFile>();
try
{
List <FstFile> srcFiles = FileSystem.Parse(fst, null, hdr.ReadString(0, 4), isGc)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
//get list of files and gaps
long end;
long gap;
long fstLen = hdr.ReadUInt32B(0x424) * (isGc ? 1L : 4L);
FstFile ff = new FstFile(null)
{
Name = "fst.bin", DataOffset = fstLen, Offset = fstLen, Length = (int)fst.Size, IsNonFstFile = true
};
for (int i = 0; i < srcFiles.Count; i++)
{
ff = i == 0 ? ff : srcFiles[i - 1];
end = ff.DataOffset + ff.Length;
end += end % 4 == 0 ? 0 : 4 - (end % 4);
gap = srcFiles[i].DataOffset - end;
if (gap < 0)
{
error = string.Format("The gap between '{0}' and '{1}' is {2} - Converting as bad image", ff.Name, srcFiles[i].Name, gap.ToString());
return(null);
}
conFiles.Add(new ConvertFile(gap, isGc)
{
FstFile = ff
});
}
ff = srcFiles.Last();
end = ff.DataOffset + ff.Length;
end += end % 4 == 0 ? 0 : 4 - (end % 4);
gap = size - end;
if (gap >= -3 && gap < 0)
{
gap = 0; //some hacked gc images converted from tgc end on the file end (star fox e3)
}
if (gap < 0)
{
error = string.Format("The gap between '{0}' and the end of the image is {1} - Converting as bad image/partition", ff.Name, gap.ToString());
return(null);
}
conFiles.Add(new ConvertFile(gap, isGc)
{
FstFile = ff
});
//set alignment
foreach (ConvertFile cf in conFiles)
{
ff = cf.FstFile;
if (fstFileAlignment == 0)
{
cf.Alignment = 0; //preserve alignment
}
else if (fstFileAlignment == -1 && ff.DataOffset % 0x8000 == 0 && (ff.Length % 0x8000 == 0 || align.Contains(Path.GetExtension(ff.Name).ToLower())))
{
cf.Alignment = 0x8000; //default behaviour
}
else if (fstFileAlignment != 0 && ff.DataOffset % fstFileAlignment == 0) //src matches alignment
{
cf.Alignment = fstFileAlignment; //align to largest multiple
}
else
{
cf.Alignment = -1; //-1 = do not align this file
}
}
}
catch
{
error = "Fst parsing error - Converting as bad image";
return(null);
}
return(conFiles);
}
internal static void NkitWriteFileSystem(Context ctx, NkitInfo imageInfo, long mlt, Stream inStream, ref long srcPos, ref long dstPos, MemorySection hdr, MemorySection fst, ref long mainDolAddr, Stream target, long nullsPos, JunkStream js, List <ConvertFile> conFiles, out List <ConvertFile> missingFiles, ScrubManager scrub, long imageSize, ILog log)
{
FstFile ff;
//########### FILES
bool firstFile = true;
ConvertFile lastf = conFiles.Last();
ConvertFile prevf = null;
missingFiles = new List <ConvertFile>();
foreach (ConvertFile f in conFiles) //read the files and write them out as goodFiles (possible order difference
{
ff = f.FstFile;
if (!firstFile)
{
if (srcPos == mainDolAddr && mainDolAddr == (hdr.ReadUInt32B(0x420) * mlt))
{
mainDolAddr = dstPos; //main/default.dol is moving
}
//Debug.WriteLine(string.Format(@"{5} : {0} : {1} : {2} : {3}/{4}", ff.DataOffset.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), /*(nextFile.DataOffset - lastEnd).ToString("X8"),*/ ff.Length.ToString("X8"), ff.Path, ff.Name, ff.OffsetInFstFile.ToString("X8")));
//srcPos aligned at 32k and the length is 32k (audio file) ensure new dest is also aligned
if (f.Alignment == 0 || (f.Alignment != -1 && dstPos % f.Alignment != 0)) //XGIII audio in race, Zelda collection games for TGC
{
long pad;
if (f.Alignment == 0) //no shrinking
{
pad = Math.Max(0, ff.DataOffset - dstPos);
}
else //align
{
pad = (int)(dstPos % f.Alignment == 0 ? 0 : f.Alignment - (dstPos % f.Alignment));
}
imageInfo.FilesAligned += 1;
ByteStream.Zeros.Copy(target, pad);
imageInfo.BytesPreservationDiscPadding += pad;
dstPos += pad;
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(dstPos / mlt));
//replace copy with junk test - adjust fst length to %4
long l = srcPos;
imageInfo.FilesTotal += 1;
long written = NkitFormat.CopyFile(ref nullsPos, f, prevf?.FstFile, target, ref srcPos, dstPos, inStream, js, imageSize, out bool missing);
if (missing)
{
missingFiles.Add(f);
}
dstPos += written;
imageInfo.BytesData += srcPos - l; //read
if (f.Gap.JunkFile != 0) //encode the gap and write - pad to %4
{
imageInfo.BytesJunkFiles += f.Gap.JunkFile + (f.Gap.JunkFile % 4 == 0 ? 0 : (4 - (f.Gap.JunkFile % 4)));
fst.WriteUInt32B(ff.OffsetInFstFile + 4, 0); //modify size to be remainder of 4
}
}
if (f.GapLength != 0 || f.Gap.JunkFile != 0)
{
long l = NkitFormat.ProcessGap(ref nullsPos, f, ref srcPos, inStream, js, firstFile || f == lastf, scrub, target, log);
imageInfo.BytesGaps += f.GapLength;
imageInfo.BytesPreservationData += l;
dstPos += l;
}
firstFile = false;
prevf = f;
}
}
internal static long CopyFile(ref long nullsPos, ConvertFile conFile, FstFile prevFile, Stream dest, ref long srcPos, long dstPos, Stream srcStream, JunkStream junkNStream, long imageSize, out bool missing)
{
//long pos = dest.Position;
missing = false;
FstFile file = conFile.FstFile;
long size = file.Length + (file.Length % 4 == 0 ? 0 : (4 - (file.Length % 4)));
if (srcPos + size > imageSize)
{
size = file.Length; //some rare GC demos end at the end of a non aligned file. This fixes it - v1.2 bugfix
}
long written = size;
byte[] f = new byte[Math.Min(0x30, size)];
srcStream.Read(f, 0, f.Length); //then read while junk is created
if (prevFile != null && prevFile.DataOffset == file.DataOffset && prevFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size
{
nullsPos = srcPos + 0x1CL; //will already be aligned
}
int nulls = (int)(nullsPos - srcPos);
int countNulls = 0;
if (f.Length > nulls)
{
junkNStream.Position = file.DataOffset; //async junk gen
for (int i = 0; i < f.Length && f[i] == 0; i++)
{
countNulls++;
}
if (countNulls < f.Length) //don't test all nulls
{
missing = junkNStream.Compare(f, 0, f.Length, Math.Max(0, countNulls)) == f.Length;
}
}
if (missing) //start of file is junk
{
//check the remainder of the file
MemorySection junkFile = MemorySection.Read(srcStream, size - f.Length);
missing = junkNStream.Compare(junkFile.Data, 0, (int)junkFile.Size, 0) == junkFile.Size;
if (missing)
{
written = 0;
conFile.Gap.SetJunkFile((uint)conFile.FstFile.Length, countNulls);
}
else //not 100% junk, write the file out
{
dest.Write(f, 0, f.Length);
dest.Write(junkFile.Data, 0, (int)junkFile.Size);
}
junkFile = null;
}
else
{
dest.Write(f, 0, f.Length);
srcStream.Copy(dest, size - f.Length); //copy file
}
if (!missing) //reset the gap when no junk
{
nullsPos = srcPos + size + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
}
srcPos += size;
return(written);
}
private bool bruteForceValidHeader(Context ctx, FstFileItem goodFst, MemorySection hdr, MemorySection bi2bin, MemorySection appldr, MemorySection maindol, MemorySection fst, Coordinator pc)
{
int hdrScrub = 0;
int bi2Scrub = 0;
int maindolScrub = 0;
foreach (int scrub in new int[] { 0, 1, 2 })
{
if (scrub == 1)
{
hdrScrub = scrubData(hdr.Data, 0x300, 4);
bi2Scrub = scrubData(bi2bin.Data, 0x500 - 0x440, 4);
if (hdrScrub == 0 && bi2Scrub == 0)
{
continue; //no change
}
}
if (scrub == 2)
{
hdrScrub = scrubData(hdr.Data, 0x80, 0x3F0 - 0x80);
bi2Scrub = scrubData(bi2bin.Data, 0x30, bi2bin.Data.Length - (0x30 + 0x60)); //0x30 at start and 0x50 from end (conflict I, II + Splinter cell have 0x40 at end)
if (maindol.Size != 0)
{
uint maindolSize = 0;
for (int i = 0; i < 18; i++)
{
if (maindol.ReadUInt32B(0x0 + (i * 4)) != 0) //7 text offsets, 11 data offsets
{
maindolSize = Math.Max(maindolSize, maindol.ReadUInt32B(0x0 + (i * 4)) + maindol.ReadUInt32B(0x90 + (i * 4)));
}
}
maindolScrub = scrubData(maindol.Data, (int)maindolSize, (int)(maindol.Data.Length - maindolSize));
}
if (hdrScrub == 0 && bi2Scrub == 0 && maindolScrub == 0)
{
continue; //no change
}
}
NCrc crc = new NCrc();
MemoryStream ms = new MemoryStream();
using (CryptoStream target = new CryptoStream(ms, crc, CryptoStreamMode.Write))
{
target.Write(hdr.Data, 0, (int)hdr.Size);
long dstPos = hdr.Data.Length;
target.Write(bi2bin.Data, 0, (int)bi2bin.Size);
dstPos += bi2bin.Size;
target.Write(appldr.Data, 0, (int)appldr.Size);
dstPos += appldr.Size;
ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), goodFst.FstOffset) - dstPos);
dstPos += Math.Min(hdr.ReadUInt32B(0x420), goodFst.FstOffset) - dstPos;
target.Write(maindol.Data, 0, Math.Min((int)(goodFst.FstOffset - dstPos), (int)maindol.Size));
dstPos += Math.Min((int)(goodFst.FstOffset - dstPos), (int)maindol.Size);
long padding = goodFst.FstOffset - dstPos;
ByteStream.Zeros.Copy(target, padding);
dstPos += padding;
target.Write(fst.Data, 0, fst.Data.Length);
dstPos += fst.Size;
Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>();
crc.Snapshot("header");
}
if (crc.FullCrc() == goodFst.PostFstCrc)
{
if (hdrScrub != 0 || bi2Scrub != 0 || maindolScrub != 0)
{
//ctx.Result.GameCube.UpdatedHeader = true;
_log?.LogDetail("Header brute forced to match");
if (scrub == 1)
{
_log?.LogDetail(string.Format(" Bytes scrubbed at 0x300 and 0x500", hdrScrub.ToString(), bi2Scrub.ToString()));
}
else if (scrub == 2)
{
_log?.LogDetail(string.Format(" Bytes scrubbed in hdr.bin ({0}), bi2.bin ({1}), main.dol ({2})", hdrScrub.ToString(), bi2Scrub.ToString(), maindolScrub.ToString()));
}
}
return(true);
}
}
return(false);
}
internal static FileSystem Parse(MemorySection ms, FstFile fst, string id)
{
return(Parse(ms, fst, id, false));
}
private long partitionWrite(Stream inStream, NCrc crc, Stream target, WiiPartitionSection pHdr, Context ctx, Coordinator pc, NkitInfo imageInfo, ScrubManager scrub, WiiHashStore hashes, long fstFileAlignment)
{
#if DEHASH
return(pHdr.NewPartitionDataLength = inStream.Copy(target, pHdr.PartitionDataLength, null));
#elif DECRYPT
inStream.Copy(target, pHdr.PartitionLength, null);
return(pHdr.NewPartitionDataLength = pHdr.PartitionDataLength);
#endif
MemorySection hdr = MemorySection.Read(inStream, 0x440);
//ProgressResult result = ctx.Result;
long mlt = 4L; //GC 1L
long logicalSize = pHdr.PartitionDataLength; //GC: ctx.ImageLength;
long physicalSize = pHdr.PartitionDataLength; //GC: result.ImageInfo.IsoSize;
string junkId = hdr.ReadString(0, 4);
//bool write = true;
List <string> addedFiles = new List <string>();
long srcPos;
long dstPos = 0;
JunkStream js = new JunkStream(junkId, hdr.Read8(6), logicalSize);
try
{
if (junkId == "\0\0\0\0")
{
srcPos = hdr.Size;
imageInfo.BytesData = srcPos;
_log?.LogDetail("Null Partition ID, preserving partition as raw");
ConvertFile cf = new ConvertFile(logicalSize - hdr.Size, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.Size, Offset = hdr.Size, Length = 0
},
};
long nullsPos = 0;
target.Write(hdr.Data, 0, (int)hdr.Size); //0x400
target.Write(pHdr.Header.Data, 0x2bc, 4); //copy the original partition length
dstPos += 0x440 + 4 + NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
hdr.WriteString(0x200, 8, "NKIT v01");
hdr.WriteUInt32B(0x210, (uint)(pHdr.PartitionLength / mlt));
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
crc.Snapshot(junkId + " PrtHdr");
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot(junkId + " Fst");
target.Write(hashes.FlagsToByteArray(), 0, (int)hashes.FlagsLength);
crc.Snapshot(junkId + " HashFlags");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos + hashes.FlagsLength;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, physicalSize, hdr, fst, false, fstFileAlignment, out error);
imageInfo.BytesData = srcPos;
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(pHdr.PartitionDataLength - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
List <ConvertFile> missing;
NkitFormat.NkitWriteFileSystem(ctx, imageInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out missing, scrub, pHdr.PartitionDataLength, _log);
dstPos += hashes.HashesToStream(target);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
crc.Crcs[crc.Crcs.Length - 1].PatchData = hashes.FlagsToByteArray();
crc.Crcs[crc.Crcs.Length - 1].PatchCrc = Crc.Compute(crc.Crcs[crc.Crcs.Length - 2].PatchData);
crc.Crcs[crc.Crcs.Length - 2].PatchData = fst.Data;
crc.Crcs[crc.Crcs.Length - 2].PatchCrc = Crc.Compute(fst.Data);
}
}
}
catch (Exception ex)
{
throw new HandledException(ex, "NkitWriterGc.Write - Convert");
}
return(dstPos);
}
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
{
try
{
WiiDiscHeaderSection hdr = null;
WiiPartitionHeaderSection pHdr = null;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
NCrc crc = new NCrc();
Crc updateCrc = new Crc();
bool updateRemoved = false;
string updateTmpFileName = null;
string updateFileName = null;
bool extractingUpdate = false;
CryptoStream updateCrcStream = null;
NStream updateTarget = null;
CryptoStream target = null;
MemorySection removedUpdateFiller = null;
int preservedHashCount = 0;
NkitInfo nkitDiscInfo = new NkitInfo();
long fstFileAlignment = -1;
WiiPartitionSection lastPart = null;
long dstPos = 0;
long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength
string ignoreJunkId;
pc.WriterCheckPoint1WriteReady(out ignoreJunkId); //wait until read has written the header and set the length
NDisc disc = new NDisc(_log, inStream);
foreach (IWiiDiscSection s in disc.EnumerateSections(imageSize)) //no fixing, image should be good //ctx.ImageLength
{
if (s is WiiDiscHeaderSection)
{
hdr = (WiiDiscHeaderSection)s;
hdr.Write8(0x60, 1);
hdr.Write8(0x61, 1);
fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == hdr.Id8)?.Item2 ?? -1;
target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(hdr.Data, 0, hdr.Data.Length); //write the header
nkitDiscInfo.BytesData += hdr.Size;
dstPos += hdr.Size;
crc.Snapshot("Disc Header");
}
else if (s is WiiPartitionSection)
{
WiiPartitionSection ps = (WiiPartitionSection)s;
WiiHashStore hashes = new WiiHashStore(ps.PartitionDataLength);
ScrubManager scrub = ps.Header.ScrubManager;
NkitInfo nkitPartInfo = new NkitInfo();
if (ps.Header.Type == PartitionType.Update && ctx.Settings.NkitUpdatePartitionRemoval && hdr.Partitions.Count() > 1) //only remove update if there's more partitions
{
updateRemoved = true;
extractingUpdate = true;
pHdr = ps.Header;
extractingUpdate = true;
updateCrcStream = new CryptoStream(ByteStream.Zeros, updateCrc, CryptoStreamMode.Write);
crc.Initialize();
nkitPartInfo.BytesData += disc.WriteRecoveryPartitionData(updateCrcStream, false, ps, 0, out updateTmpFileName, out updateFileName, out updateTarget);
_log?.LogDetail(string.Format("Extracted and Removed {0} Recovery Partition: [{1}]", ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));
removedUpdateFiller = new MemorySection(new byte[0x8000]);
removedUpdateFiller.Write(0, hdr.Read(0x40000, 0x100)); //backup the original partition table in case it's non standard in some way
}
else
{
target.Write(ps.Header.Data, 0, (int)ps.Header.Size);
nkitDiscInfo.BytesData += ps.Header.Size;
crc.Snapshot(ps.Id + " Hdr");
crc.Crcs.Last().PatchData = ps.Header.Data;
dstPos += ps.Header.Size;
ps.NewDiscOffset = dstPos;
//long written = 0;
using (StreamCircularBuffer decrypted = new StreamCircularBuffer(ps.PartitionDataLength, null, null, output =>
{
//read decrypted partition
foreach (WiiPartitionGroupSection pg in ps.Sections)
{
nkitPartInfo.BytesHashesData += (pg.Size / 0x8000 * 0x400); //size of input hashes
if (pg.PreserveHashes())
{
nkitPartInfo.BytesHashesPreservation += hashes.Preserve(pg.Offset, pg.Decrypted, pg.Size);
if (++preservedHashCount >= 1500) //too many will bomb the patch caching when converting back. Something is wrong
{
throw pc.SetWriterException(new HandledException("Over 1500 hashes preserved, aborting as image is corrupt or poorly scrubbed."));
}
}
#if !DECRYPT
for (int i = 0; i < pg.Size / 0x8000; i++)
{
output.Write(pg.Decrypted, (i * 0x8000) + 0x400, 0x7c00);
}
#else
output.Write(pg.Decrypted, 0, (int)pg.Size);
#endif
}
}))
{
long len = partitionWrite(decrypted, crc, target, ps, ctx, pc, nkitPartInfo, scrub, hashes, fstFileAlignment);
ps.NewPartitionDataLength = len;
dstPos += len;
lastPart = ps;
}
}
NkitFormat.LogNkitInfo(nkitPartInfo, _log, ps.Id, false);
lastPartitionId = ps.Id;
lastPartitionType = ps.Header.Type;
}
else if (s is WiiFillerSection)
{
WiiFillerSection fs = (WiiFillerSection)s;
ScrubManager scrub = new ScrubManager(null);
JunkStream junk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
if (lastPartitionType == PartitionType.Update && updateRemoved)
{
//preserve the original partition table and update filename
target.Write(removedUpdateFiller.Data, 0, (int)removedUpdateFiller.Size); //remove update partition by adding a 32k placeholder. Avoid having a non update partition at 0x50000
nkitDiscInfo.BytesPreservationDiscPadding += removedUpdateFiller.Size;
nkitDiscInfo.BytesGaps += fs.Size;
dstPos += removedUpdateFiller.Size;
crc.Snapshot(string.Format("{0}Replacement Filler", string.IsNullOrEmpty(lastPartitionId) ? "" : (" " + SourceFiles.CleanseFileName(lastPartitionId).PadRight(4))));
if (extractingUpdate)
{
int storeType;
if ((storeType = disc.WriteRecoveryPartitionFiller(updateCrcStream, junk, fs.DiscOffset, true, true, fs, updateTarget, updateTmpFileName, ref updateFileName, updateCrc, true)) != 0)
{
_log.LogDetail(string.Format("{0}Update recovery partition stored: {1}", storeType == 2 ? "Other " : "", updateFileName));
}
extractingUpdate = false;
}
}
else
{
if (fs.Size != 0)
{
using (StreamCircularBuffer filler = new StreamCircularBuffer(fs.Size, null, null, output =>
{
foreach (WiiFillerSectionItem item in ((WiiFillerSection)s).Sections)
{
output.Write(item.Data, 0, (int)item.Size);
}
}))
{
Gap gap = new Gap(fs.Size, false);
long srcPos = fs.DiscOffset;
long gapLen = gap.Encode(filler, ref srcPos, lastPartitionType == PartitionType.Update ? fs.Size : 0x1cL, fs.Size, junk, scrub, target, _log);
nkitDiscInfo.BytesPreservationData += gapLen;
nkitDiscInfo.BytesGaps += fs.Size;
dstPos += gapLen;
if (lastPart != null)
{
lastPart.NewPartitionDataLength += gapLen;
}
}
}
//pad partition to 32k
int gapLen2 = (int)(dstPos % 0x8000 == 0 ? 0 : 0x8000 - (dstPos % 0x8000));
ByteStream.Zeros.Copy(target, gapLen2);
nkitDiscInfo.BytesPreservationDiscPadding += gapLen2;
dstPos += gapLen2;
if (lastPart != null)
{
lastPart.NewPartitionDataLength += gapLen2;
}
if (lastPart != null)
{
lastPart.Header.WriteUInt32B(0x2bc, (uint)(lastPart.NewPartitionDataLength / 4)); //updates the array in the crc data
hdr.Partitions.First(a => a.DiscOffset == lastPart.DiscOffset).DiscOffset = (lastPart.NewDiscOffset - 0x20000);
}
crc.Snapshot(string.Format("{0}{1}Files+Filler", lastPartitionId ?? "", string.IsNullOrEmpty(lastPartitionId) ? "" : " "));
}
}
}
NkitFormat.LogNkitInfo(nkitDiscInfo, _log, hdr.Id, true);
foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
{
ci.PatchCrc = Crc.Compute(ci.PatchData);
}
NCrc readerCrcs;
uint validationCrc;
pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.
if (updateRemoved && crc.Crcs.Length > 2) //freeloader wii only has update partition
{
hdr.RemoveUpdatePartition(crc.Crcs[2].Offset);
}
hdr.UpdateOffsets();
hdr.WriteString(0x200, 8, "NKIT v01"); //header and version
hdr.Write8(0x60, 1);
hdr.Write8(0x61, 1);
hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
hdr.WriteUInt32B(0x210, (uint)(imageSize / 4L)); //ctx.ImageLength
hdr.WriteUInt32B(0x218, updateCrc.Value); //Update crc - Only if removed
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); //update with magic applied
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "NKit Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "NkitWriterWii.Write - Convert");
}
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
string resultMsg = "";
if (!Settings.ConfigFileFound)
{
_log?.Log("!! No config file found - This is required to restore and validate images");
}
NCrc crc = new NCrc();
Settings settings = ctx.Settings;
List <string> addedFiles = new List <string>();
DatData data = ctx.Dats;
RecoveryData rec = ctx.Recovery;
List <JunkRedumpPatch> patches = rec.JunkPatches;
MemorySection hdr = null;
try
{
long junkStart = settings.JunkStartOffsets.FirstOrDefault(a => a.Id8 == inStream.Id8)?.Offset ?? 0;
string forceJunkId = settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == inStream.Id8)?.JunkId;
if (forceJunkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", forceJunkId, inStream.Id));
//result.ImageInfo.JunkId = forceJunkId;
inStream.ChangeJunk(forceJunkId);
}
if (junkStart != 0)
{
_log?.LogDetail(string.Format("Junk forced to start at 0x{0}", junkStart.ToString("X8")));
}
hdr = inStream.DiscHeader;
FstFileItem goodFst = rec.GcBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8 && !(inStream.Id8 == "GNHE5d0000" && a.Length == 116));
if (goodFst != null)
{
_log?.LogDetail(string.Format("Recovery: {0}", goodFst.Filename));
}
if (goodFst == null && rec.GcNewBinFiles != null)
{
goodFst = rec.GcNewBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8);
}
if (goodFst != null)
{
goodFst.Populate();
}
ApploaderFileItem goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
if (goodAldr == null && rec.GcNewBinFiles != null)
{
goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcNewBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
}
//is it an action replay (custom hacks)
if ((hdr.ReadUInt32B(0x420) == 0 && inStream.Id8 == "GNHE5d0000") || inStream.Id8 == "DTLX010000" || inStream.Id8 == "102E010007")
{
if (inStream.Id8 == "102E010007")
{
resultMsg = "Aging Disc detected - Skipping recover";
}
else
{
resultMsg = "Datel Action Replay detected - Skipping recover";
}
try
{
using (CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write))
{
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
inStream.Copy(target, pc.OutputSize - hdr.Size);
}
_log?.LogDetail(resultMsg);
}
catch (Exception ex)
{
throw new HandledException(ex, resultMsg);
}
crc.Snapshot("files");
}
else
{
//############################################################################
//# READ DISC START
//########### Header (boot.bin) 0 (read by base stream already)
long srcPos = hdr.Size;
_log?.LogDetail(string.Format("Header Read: {0} - {1}", friendly(inStream.Id), friendly(inStream.Title)));
//########### Header Info (bi2.bin) 0x440
MemorySection bi2bin = MemorySection.Read(inStream, 0x2000);
srcPos += bi2bin.Size;
//########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol
MemorySection appldr = MemorySection.Read(inStream, Math.Min(hdr.ReadUInt32B(0x420) == 0 ? uint.MaxValue : hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - srcPos);
srcPos += appldr.Size;
//########### APP (main.dol)
MemorySection maindol;
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424))
{
maindol = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - srcPos);
}
else
{
maindol = new MemorySection(new byte[0]);
}
srcPos += maindol.Size;
//########### FST (fst.bin)
MemorySection srcFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428));
srcPos += hdr.ReadUInt32B(0x428);
//############################################################################
//# CORRECT ISSUES
List <FstFile> srcFiles = FileSystem.Parse(srcFst, null, inStream.Id, true)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
if (srcFiles == null)
{
throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", hdr.ReadUInt32B(0x424).ToString("X8")));
}
uint crcTmp = Crc.Compute(appldr.Data, 0, Math.Min((int)appldr.Size, 0x20 + (int)(appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18))));
//adjust appldr
if (goodAldr != null && crcTmp != goodAldr.Crc)
{
_log?.LogDetail(string.Format("Replacing appldr.bin crc {0} with Recovery appldr.bin {1}", crcTmp.ToString("X8"), goodAldr.Crc.ToString("X8")));
addedFiles.Add(goodAldr.Filename);
appldr = new MemorySection(File.ReadAllBytes(goodAldr.Filename));
}
//adjust main.dol
if (goodFst != null)
{
if (goodFst.MainDolOffset != hdr.ReadUInt32B(0x420))
{
//main.dol is before fst in image and after in goodfst
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424) && goodFst.MainDolOffset > goodFst.FstOffset)
{
_log?.LogDetail(string.Format("Skipping main.dol at 0x{0} using 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
maindol = new MemorySection(new byte[0]);
}
else
{
_log?.LogDetail(string.Format("Moving main.dol address 0x{0} to 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
}
hdr.WriteUInt32B(0x420, (uint)goodFst.MainDolOffset);
}
if (goodFst.Region != (Region)bi2bin.ReadUInt32B(0x18))
{
_log?.LogDetail(string.Format("Region Changed to {0} from {1}", goodFst.Region.ToString(), ((Region)bi2bin.ReadUInt32B(0x18)).ToString()));
bi2bin.WriteUInt32B(0x18, (uint)goodFst.Region);
}
if (goodFst.MaxFst != hdr.ReadUInt32B(0x42C))
{
_log?.LogDetail(string.Format("Max Fst Size changed to {0} from {1}", goodFst.MaxFst.ToString("X8"), hdr.ReadUInt32B(0x42C).ToString("X8")));
hdr.WriteUInt32B(0x42C, (uint)goodFst.MaxFst);
}
string newTitle = Encoding.ASCII.GetString(goodFst.Title);
if (newTitle != hdr.ReadString(0x20, 0x60 - 0x20))
{
_log?.LogDetail(string.Format("Title changed to '{0}' from '{1}'", newTitle.TrimEnd('\0'), hdr.ReadString(0x20, 0x60 - 0x20).TrimEnd('\0')));
hdr.Write(0x20, goodFst.Title);
}
}
MemorySection fst = srcFst;
List <FstFile> fstFiles = srcFiles;
crcTmp = Crc.Compute(srcFst.Data);
if (goodFst != null && crcTmp != goodFst.Crc)
{
_log?.LogDetail(string.Format("Replacing fst.bin crc {0} with Recovery fst {1}", crcTmp.ToString("X8"), goodFst.Crc.ToString("X8")));
addedFiles.Add(goodFst.Filename);
fst = new MemorySection(goodFst.FstData);
fstFiles = FileSystem.Parse(fst, null, inStream.Id, true).Files.OrderBy(a => a.Offset).ThenBy(a => a.Length).ToList();
}
//adjust fst.bin
if (goodFst != null && (goodFst.FstOffset != hdr.ReadUInt32B(0x424) || goodFst.FstData.Length != hdr.ReadUInt32B(0x428)))
{
_log?.LogDetail(string.Format("Moving / Resizing fst.bin address 0x{0} (Length {1}) to 0x{2} (Length {3})", hdr.ReadUInt32B(0x424).ToString("X8"), hdr.ReadUInt32B(0x428).ToString(), goodFst.FstOffset.ToString("X8"), goodFst.FstData.Length.ToString()));
hdr.WriteUInt32B(0x424, (uint)goodFst.FstOffset);
hdr.WriteUInt32B(0x428, (uint)goodFst.FstData.Length);
}
int c;
int failCount = 0;
int filesMoved = 0;
int nkitJunkFiles = 0;
foreach (FstFile f in srcFiles)
{
FstFile[] fnd = fstFiles.Where(a => a.Name == f.Name && a.Path == f.Path).ToArray();
if (fnd.Length == 0)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: No File Found - {0}/{1} (Length {2})", f.Path, f.Name, f.Length.ToString()));
}
else if (inStream.IsNkit && (c = fnd.Count(a => a.Length != 0 && f.Length == 0)) == 1)
{
nkitJunkFiles++;
}
else if ((c = fnd.Count(a => a.Length == f.Length)) != 1)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: File Size bad - {0}/{1} (Length {2}){3}", f.Path, f.Name, f.Length.ToString(), c <= 1 ? "" : string.Format(" {0} files found", c.ToString())));
}
else if (f.DataOffset != fnd[0].DataOffset)
{
filesMoved++;
}
if (failCount >= 10)
{
break;
}
}
if (failCount != 0)
{
if (failCount >= 10)
{
throw new HandledException(string.Format("{0} or more FST errors found", failCount.ToString()));
}
}
if (filesMoved != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be repositioned when rebuilding this image", filesMoved.ToString(), filesMoved == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (nkitJunkFiles != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be generated from junk when rebuilding this image", nkitJunkFiles.ToString(), nkitJunkFiles == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (goodFst != null)
{
//is the data output so far correct
if (!bruteForceValidHeader(ctx, goodFst, hdr, bi2bin, appldr, maindol, fst, pc))
{
throw new HandledException(string.Format("Post FST Crc Failed 0x{0}", goodFst.PostFstCrc.ToString("X8")));
}
}
//############################################################################
//# WRITE DISC START
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
long dstPos = hdr.Data.Length;
crc.Snapshot("boot.bin");
target.Write(bi2bin.Data, 0, (int)bi2bin.Size);
dstPos += bi2bin.Size;
crc.Snapshot("bi2.bin");
target.Write(appldr.Data, 0, (int)appldr.Size);
dstPos += appldr.Size;
ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos);
dstPos += Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos;
crc.Snapshot("appldr.bin");
target.Write(maindol.Data, 0, (int)maindol.Size);
dstPos += maindol.Size;
if (goodFst != null)
{
long padding = goodFst.FstOffset - dstPos;
ByteStream.Zeros.Copy(target, padding);
dstPos += padding;
}
crc.Snapshot("main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
dstPos += fst.Size;
bool firstFile = true;
Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>();
crc.Snapshot("fst.bin");
//############################################################################
//# WRITE THE FILESYSTEM
int fidx = -1;
FstFile lastFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = fst.Size
};
FstFile nextFile = fstFiles[++fidx];
FstFile cacheFile = null;
long nullsPos = dstPos + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
//########### FILES
foreach (FstFile f in srcFiles) //read the files and write them out as goodFiles (possible order difference)
{
while ((cacheFile = cache.Keys.FirstOrDefault(a => (a.Length == nextFile.Length || (inStream.IsNkit && a.Length == 0)) && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) //write cache
{
bool isJunk = inStream.IsNkit && cacheFile.Length == 0 && cacheFile.Name == nextFile.Name && cacheFile.Path == nextFile.Path;
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, isJunk);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
if (f.DataOffset - srcPos > 0) //skip src junk
{
inStream.Copy(ByteStream.Zeros, f.DataOffset - srcPos);
srcPos += f.DataOffset - srcPos;
}
if (nextFile != null && (f.Length != nextFile.Length || f.Name != nextFile.Name || f.Path != nextFile.Path)) //cache file (nkit junk files are cached)
{
bool isNkitJunk = inStream.IsNkit && f.Length == 0 && nextFile.Length != 0 && f.Name == nextFile.Name && f.Path == nextFile.Path;
byte[] cacheItem = new byte[isNkitJunk ? 8 : f.Length];
inStream.Read(cacheItem, 0, cacheItem.Length); //read the nkit junk data - details (real length and null counts)
cache.Add(f, cacheItem);
srcPos += cacheItem.Length;
}
else //copy file
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, inStream, lastFile, nextFile, patches, inStream, junkStart, false);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
srcPos += f.Length;
}
}
while (nextFile != null && fidx < fstFiles.Count && (cacheFile = cache.Keys.FirstOrDefault(a => a.Length == nextFile.Length && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null)
{
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, false);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
writeDestGap(nullsPos, target, dstPos, pc.OutputSize - dstPos, true, patches, junkStart, inStream);
crc.Snapshot("files");
}
resultMsg = "MatchFail";
uint finalCrc = crc.FullCrc(true);
if (ctx.Dats.RedumpData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Redump";
}
if (ctx.Dats.CustomData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Custom";
}
pc.ReaderCheckPoint2Complete(crc, false, finalCrc, finalCrc, VerifyIsWrite, hdr.Data, resultMsg);
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "RestoreReaderGc.Read - Read and repair"); //don't let the writer lock
}
}
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
{
try
{
long mlt = 1L; //for Wii: 4L
long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength
pc.WriterCheckPoint1WriteReady(out string junkId); //wait until read has written the header and set the length
List <string> addedFiles = new List <string>();
NCrc crc = new NCrc();
long srcPos;
long dstPos = 0;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
string id8 = string.Concat(hdr.ReadString(0, 6), hdr.Data[6].ToString("X2"), hdr.Data[7].ToString("X2"));
if (junkId == null)
{
junkId = ctx.Settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == id8)?.JunkId;
if (junkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", junkId, id8.Substring(0, 4)));
}
}
if (junkId == null)
{
junkId = hdr.ReadString(0, 4);
}
MemorySection fst;
List <JunkDiff> junkDiffs = new List <JunkDiff>();
long mainDolAddr = hdr.ReadUInt32B(0x420) * mlt;
long fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == id8)?.Item2 ?? -1;
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
//############################################################################
//# READ DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
crc.Snapshot("hdr.bin");
inStream.Copy(target, (hdr.ReadUInt32B(0x424) * mlt) - hdr.Size);
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
//read fst with 4 byte boundary
fst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x428) * mlt) + (((hdr.ReadUInt32B(0x428) * mlt) % 4 == 0 ? 0 : 4 - ((hdr.ReadUInt32B(0x428) * mlt) % 4))));
target.Write(fst.Data, 0, (int)fst.Size);
crc.Snapshot("fst.bin");
srcPos = (hdr.ReadUInt32B(0x424) * mlt) + fst.Size;
long nullsPos = srcPos + 0x1c;
dstPos = srcPos;
//create as late as possible in case id is swaped - Dairantou Smash Brothers DX (Japan) (Taikenban), Star Wars - Rogue Squadron II (Japan) (Jitsuen-you Sample)
JunkStream js = new JunkStream(junkId, hdr.Read8(6), NStream.FullSizeGameCube);
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, imageSize, hdr, fst, true, fstFileAlignment, out string error); //Size isn't important for writing //result.ImageInfo.IsoSize
NkitInfo nkitInfo = new NkitInfo
{
BytesData = srcPos,
BytesGaps = 0,
BytesJunkFiles = 0,
BytesPreservationData = 0,
BytesPreservationDiscPadding = 0
};
ScrubManager scrub = new ScrubManager();
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //Size isn't important for writing //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
NkitFormat.ProcessGap(ref nullsPos, cf, ref srcPos, inStream, js, true, scrub, target, _log);
}
else
{
//############################################################################
//# WRITE THE FILESYSTEM
NkitFormat.NkitWriteFileSystem(ctx, nkitInfo, mlt, inStream, ref srcPos, ref dstPos, hdr, fst, ref mainDolAddr, target, nullsPos, js, conFiles, out List <ConvertFile> missing, scrub, imageSize, _log);
if (missing.Count != 0)
{
_log?.LogDetail(string.Format("{0} Junk File{1} Removed (Files listed in the FST, but not in the image)", missing.Count.ToString(), missing.Count == 1 ? "" : "s"));
foreach (ConvertFile cf in missing)
{
_log?.LogDebug(string.Format("File content is Junk {0}: {1} - Size: {2}", cf.FstFile.DataOffset.ToString("X8"), cf.FstFile.Name, cf.FstFile.Length));
}
}
}
if (dstPos % 0x800 != 0)
{
long l = 0x800 - (dstPos % 0x800);
ByteStream.Zeros.Copy(target, l);
dstPos += l;
nkitInfo.BytesPreservationDiscPadding += l;
}
crc.Snapshot("files");
NkitFormat.LogNkitInfo(nkitInfo, _log, hdr.ReadString(0, 4), true);
pc.WriterCheckPoint2Complete(out NCrc readerCrcs, out uint validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.
hdr.WriteUInt32B(0x420, (uint)mainDolAddr);
hdr.WriteString(0x200, 8, "NKIT v01"); //header and version
hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
hdr.WriteUInt32B(0x210, (uint)imageSize); //result.ImageInfo.IsoSize);
hdr.WriteString(0x214, 4, hdr.ReadString(0, 4) != junkId ? junkId : "\0\0\0\0");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), VerifyIsWrite, "NKit Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "NkitWriterGc.Write - Convert");
}
}
private long writeFile(ref long nullsPos, Stream dest, long dstPos, ref bool firstFile, Stream srcStream, FstFile lastFile, FstFile file, List <JunkRedumpPatch> patches, NStream nstream, long junkStart, bool isJunkFile)
{
//Debug.WriteLine(string.Format(@"{0} : {1} : {2} : {3}/{4}", file.DataOffset.ToString("X8"), (file.DataOffset + file.Length).ToString("X8"), /*(nextFile.DataOffset - lastEnd).ToString("X8"),*/ file.Length.ToString("X8"), file.Path, file.Name));
//file found
long gap = writeDestGap(nullsPos, dest, dstPos, file.DataOffset - dstPos, firstFile, patches, junkStart, nstream);
firstFile = false;
bool missing = false;
if (file.Length == 0)
{
// missing = true;
}
else if (isJunkFile)
{
missing = true;
MemorySection ms = MemorySection.Read(srcStream, 8);
long size = ms.ReadUInt32B(0);
GapType gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
//set nullsPos value if zerobyte file without junk
if (gt == GapType.JunkFile)
{
nullsPos = Math.Min(nullsPos - (dstPos + gap), 0); //reset the nulls
long nulls = (size & 0xFC) >> 2;
long junkFileLen = ms.ReadUInt32B(4);
if (junkFileLen != file.Length)
{
throw new HandledException(string.Format("NKit Junk file restoration length mismatch {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
ByteStream.Zeros.Copy(dest, nulls);
nstream.JunkStream.Position = (dstPos + gap) + nulls;
nstream.JunkStream.Copy(dest, junkFileLen - nulls);
_log?.LogDetail(string.Format("Generated file content with Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
else
{
throw new HandledException(string.Format("NKit Junk file restoration bytes invalid {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
}
else
{
byte[] f = new byte[Math.Min(0x30, file.Length)];
srcStream.Read(f, 0, f.Length); //then read while junk is created
if (lastFile.DataOffset == file.DataOffset && lastFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size
{
nullsPos = dstPos + 0x1CL; //will already be aligned
}
int nulls = (int)(nullsPos - (dstPos + gap));
nstream.JunkStream.Position = file.DataOffset; //async junk gen
int countNulls = 0;
for (int i = 0; i < f.Length && f[i] == 0; i++)
{
countNulls++;
}
if (f.Length > nulls && countNulls < f.Length) //don't test all nulls
{
missing = nstream.JunkStream.Compare(f, 0, f.Length, Math.Max(0, nulls)) == f.Length;
}
if (missing)
{
_log?.LogDetail(string.Format("File content is Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
dest.Write(f, 0, f.Length);
srcStream.Copy(dest, file.Length - f.Length); //copy file
}
if (!missing) //reset the gap when no junk
{
nullsPos = dstPos + gap + file.Length + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
}
return(gap + file.Length);
}
public void Write(Context ctx, Stream input, Stream output, Coordinator pc)
{
try
{
long imageSize = pc.OutputSize;
string junkId;
pc.WriterCheckPoint1WriteReady(out junkId); //wait until read has written the header and set the length
MemorySection ms = new MemorySection(new byte[4 + 4 + 8 + 8 + 4 + 4]);
int blockSize = 0x4000;
int blocks = (int)(imageSize / blockSize) + (imageSize % blockSize == 0 ? 0 : 1);
ms.WriteUInt32L(0x00, 0xB10BC001);
ms.WriteUInt32L(0x04, 0); //insert NKIT later
ms.WriteUInt64L(0x08, 0); //size place holder
ms.WriteUInt64L(0x10, (ulong)imageSize);
ms.WriteUInt32L(0x18, (uint)blockSize);
ms.WriteUInt32L(0x1C, (uint)blocks);
MemorySection pnt = new MemorySection(new byte[blocks * 8]);
MemorySection hsh = new MemorySection(new byte[blocks * 4]);
long offset = 0;
NCrc crc = new NCrc();
CryptoStream target = new CryptoStream(output, crc, CryptoStreamMode.Write);
target.Write(ms.Data, 0, (int)ms.Size);
crc.Snapshot("Header");
target.Write(pnt.Data, 0, (int)pnt.Size);
crc.Snapshot("Pointers");
target.Write(hsh.Data, 0, (int)hsh.Size);
crc.Snapshot("Hashes");
long dataOffset = (pnt.Size + hsh.Size + ms.Size); //fso.position
int rdBlk = 0;
int wrBlk = 0;
object rdBlkLock = new object();
object wrBlkLock = new object();
Task[] tasks = new Task[3];
ManualResetEvent mr = new ManualResetEvent(false);
for (int i = 0; i < tasks.Length; i++) //4 threads
{
tasks[i] = (new TaskFactory()).StartNew((Object prm) =>
{
ManualResetEvent m = (ManualResetEvent)prm;
int blkIdx;
byte[] rawBlk = new byte[blockSize];
byte[] buff = new byte[blockSize + 0x100];
using (MemoryStream blk = new MemoryStream(buff))
{
while (true)
{
lock (rdBlkLock)
{
blkIdx = rdBlk++;
if (blkIdx >= blocks)
{
return; //no more processing
}
int read = input.Read(rawBlk, 0, (int)Math.Min(imageSize - (blkIdx * blockSize), blockSize));
if (read != blockSize)
{
Array.Clear(rawBlk, read, blockSize - read);
}
}
blk.Position = 0;
ZlibStream zl = new ZlibStream(blk, SharpCompress.Compressors.CompressionMode.Compress, SharpCompress.Compressors.Deflate.CompressionLevel.BestCompression, Encoding.Default);
zl.FlushMode = FlushType.Finish;
zl.Write(rawBlk, 0, blockSize);
zl.Flush();
while (blkIdx != wrBlk)
{
m.WaitOne();
}
mr.Reset();
if (blk.Position < blockSize)
{
target.Write(buff, 0, (int)blk.Position);
pnt.WriteUInt64L(blkIdx * 8, (ulong)(offset));
hsh.WriteUInt32L(blkIdx * 4, adler32(buff, (int)blk.Position));
offset += (int)blk.Position;
}
else
{
target.Write(rawBlk, 0, blockSize);
pnt.WriteUInt64L(blkIdx * 8, (ulong)(offset) | 0x8000000000000000);
hsh.WriteUInt32L(blkIdx * 4, adler32(rawBlk, blockSize));
offset += (int)blockSize;
}
target.Flush();
wrBlk++;
mr.Set();
}
}
}, mr);
}
Task.WaitAll(tasks);
crc.Snapshot("Files");
NCrc readerCrcs;
uint validationCrc;
pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, null, dataOffset + offset); //wait until reader has completed and get crc patches.
ms.WriteUInt64L(0x08, (ulong)offset);
crc.Crcs[0].PatchCrc = Crc.Compute(ms.Data);
crc.Crcs[0].PatchData = ms.Data;
crc.Crcs[1].PatchCrc = Crc.Compute(pnt.Data);
crc.Crcs[1].PatchData = pnt.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(hsh.Data);
crc.Crcs[2].PatchData = hsh.Data;
ms.WriteUInt32B(0x04, CrcForce.Calculate(crc.FullCrc(true), output.Length, readerCrcs.FullCrc(true), 0x04, ms.ReadUInt32B(0x04))); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(ms.Data);
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "GCZ Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "GczWriter.Write - Compress");
}
}
