private long writeFiller(ref long srcPos, long dstPos, long nullsPos, Stream inStream, Stream target, JunkStream junk, ScrubManager scrub)
{
long fileLength = -1; //will be ignored
LongRef gapLength = new LongRef()
{
Value = -1
};
dstPos = writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, dstPos, inStream, target, junk, true, scrub);
return(dstPos);
}
private long writeGap(ConvertFile file, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target, JunkStream junk, bool firstOrLastFile, ScrubManager scrub)
{
long fileLength = file.FstFile.Length;
LongRef gapLength = new LongRef()
{
Value = file.GapLength
};
dstPos = writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, dstPos, inStream, target, junk, firstOrLastFile, scrub);
file.FstFile.Length = fileLength;
file.GapLength = gapLength.Value;
return(dstPos);
}
private long writeGap(ref long fileLength, LongRef gapLength, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target, JunkStream junk, bool firstOrLastFile, ScrubManager scrub)
{
if (gapLength.Value == 0)
{
if (fileLength == 0)
{
nullsPos = dstPos + 0x1c;
}
return(0);
}
long srcLen = gapLength.Value; //fix added for (padding between junk files) - Zumba Fitness (Europe) (En,Fr,De,Es,It)
MemorySection ms = MemorySection.Read(inStream, 4);
srcPos += 4;
long size = ms.ReadUInt32B(0);
GapType gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
if (size == 0xFFFFFFFC) //for wii only. not a thing for GC
{
srcPos += 4;
inStream.Read(ms.Data, 0, 4);
size = 0xFFFFFFFCL + (long)ms.ReadUInt32B(0); //cater for files > 0xFFFFFFFF
}
gapLength.Value = size;
scrub.AddGap(fileLength, dstPos, size); //keep track of trailing nulls when restoring scrubbed images
long nulls;
long junkFileLen = 0;
//set nullsPos value if zerobyte file without junk
if (gt == GapType.JunkFile)
{
nullsPos = Math.Min(nullsPos - dstPos, 0);
nulls = (size & 0xFC) >> 2;
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
junkFileLen = ms.ReadUInt32B(0);
fileLength = junkFileLen;
junkFileLen += junkFileLen % 4 == 0 ? 0 : 4 - (junkFileLen % 4);
ByteStream.Zeros.Copy(target, nulls);
junk.Position = dstPos + nulls;
junk.Copy(target, junkFileLen - nulls);
dstPos += junkFileLen;
if (srcLen <= 8)
{
return(junkFileLen);
}
else
{
//read gap
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
size = ms.ReadUInt32B(0);
gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
gapLength.Value = size;
}
}
else if (fileLength == 0) //last zero byte file was legit
{
nullsPos = dstPos + 0x1c;
}
long maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (size < maxNulls) //need to test this commented if
{
nulls = size;
}
else
{
nulls = size >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
nullsPos = dstPos + nulls; //belt and braces
if (gt == GapType.AllJunk)
{
ByteStream.Zeros.Copy(target, nulls);
junk.Position = dstPos + nulls;
junk.Copy(target, size - nulls);
dstPos += size;
}
else if (gt == GapType.AllScrubbed)
{
scrub.Scrub(target, dstPos, size, 0);
dstPos += size;
}
else
{
long prg = size;
byte btByte = 0x00;
GapBlockType bt = GapBlockType.Junk; //should never be used
while (prg > 0)
{
inStream.Read(ms.Data, 0, 4);
srcPos += 4;
long bytes;
long blk = ms.ReadUInt32B(0);
GapBlockType btType = (GapBlockType)(blk >> 30);
bool btRepeat = btType == GapBlockType.Repeat;
if (!btRepeat)
{
bt = btType;
}
long cnt = 0x3FFFFFFF & blk;
if (bt == GapBlockType.NonJunk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
inStream.Copy(target, bytes);
srcPos += bytes;
}
else if (bt == GapBlockType.ByteFill)
{
if (!btRepeat)
{
btByte = (byte)(0xFF & cnt); //last 8 bits when not repeating are the byte
cnt >>= 8;
}
bytes = Math.Min(cnt * Gap.BlockSize, prg);
scrub.Scrub(target, dstPos, bytes, btByte);
}
else //if (bt == GapBlockType.Junk)
{
bytes = Math.Min(cnt * Gap.BlockSize, prg);
maxNulls = Math.Max(0, nullsPos - dstPos); //0x1cL
if (prg < maxNulls)
{
nulls = bytes;
}
else
{
nulls = bytes >= 0x40000 && !firstOrLastFile ? 0 : maxNulls;
}
ByteStream.Zeros.Copy(target, nulls);
junk.Position = dstPos + nulls;
junk.Copy(target, bytes - nulls);
}
prg -= bytes;
dstPos += bytes;
}
}
return(gapLength.Value + junkFileLen);
}
private long partitionStreamWrite(LongRef outSize, Stream inStream, Stream target, long size, DatData settingsData, NkitPartitionPatchInfo patchInfo, WiiHashStore hashes, Coordinator pc)
{
DatData data = settingsData;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
long srcPos = hdr.Size;
long outPos = 0;
long imageSize = 0;
try
{
if (hdr.ReadString(0, 4) == "\0\0\0\0")
{
long nullsPos = 0;
long fileLength = -1;
LongRef gapLength = new LongRef()
{
Value = -1
};
target.Write(hdr.Data, 0, (int)hdr.Size);
MemorySection sz = MemorySection.Read(inStream, 4);
srcPos += 4;
outPos += hdr.Size;
imageSize = sz.ReadUInt32B(0) * 4L;
outSize.Value = NStream.HashedLenToData(imageSize);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), outSize.Value); //SET LENGTH FROM HEADER
outPos += writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
imageSize = NStream.HashedLenToData((hdr.ReadUInt32B(0x210) * 4L));
outSize.Value = imageSize;
string junkId = hdr.ReadString(0x214, 4);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), imageSize); //SET LENGTH FROM HEADER
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x424) * 4L) - hdr.Size);
srcPos += hdrToFst.Size;
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) * 4L);
long postFstPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
srcPos += fst.Size;
hashes.WriteFlagsData(imageSize, inStream);
srcPos += hashes.FlagsLength;
patchInfo.PartitionDataHeader = hdr;
patchInfo.Fst = fst;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
target.Write(fst.Data, 0, fst.Data.Length);
hdrToFst = null; //let this be collected if needed
outPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
long nullsPos = outPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, size, hdr, fst, false, -1, out error);
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
outPos += writeGap(cf, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
conFiles[0].GapLength -= hashes.FlagsLength; //fix for a few customs (no gap between the fst and the first file on the source image, but the hash mask makes it look like there is)
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), outPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (outPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset) //skip any padding (not written for wii currently)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)(outPos / 4L));
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(outPos / 4L));
outPos += copyFile(f, ref nullsPos, ref srcPos, outPos, inStream, target);
}
if (outPos < imageSize)
{
long gapLen = writeGap(f, ref nullsPos, ref srcPos, outPos, inStream, target, junk, i == 0 || i == conFiles.Count - 1, patchInfo.ScrubManager);
outPos += gapLen;
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)(ff.Length));
}
}
firstFile = false;
}
}
}
return(srcPos);
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - partitionRead");
}
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
WiiDiscHeaderSection hdr = (WiiDiscHeaderSection)inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
long imageSize = hdr.ReadUInt32B(0x210) * 4L;
string junkId = hdr.ReadString(0x214, 4);
uint updatePartitionCrc = hdr.ReadUInt32B(0x218);
MemorySection updateRemovedFiller = null;
long discOffset = 0;
List <NkitPartitionPatchInfo> patchInfos = new List <NkitPartitionPatchInfo>();
discOffset += hdr.Size;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
NCrc crc = new NCrc();
long dstPos = 0;
long srcPos = hdr.Size;
ScrubManager scrubFiller = new ScrubManager(null);
bool isRecoverable = false;
using (NDisc disc = new NDisc(_log, inStream))
{
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
hdr.Write8(0x60, 0);
hdr.Write8(0x61, 0);
CryptoStream crcStream = new CryptoStream(outStream, crc, CryptoStreamMode.Write); //wrap to calculate crc
crcStream.Write(hdr.Data, 0, hdr.Data.Length); //write the header
pc.ReaderCheckPoint1PreWrite(null, nkitCrc); //size that will be output from this read
dstPos += hdr.Size;
crc.Snapshot("Disc Header");
foreach (WiiPartitionInfo part in hdr.Partitions) //already sorted
{
if (updatePartitionCrc != 0 && srcPos == hdr.Size) //write update partition out
{
updateRemovedFiller = MemorySection.Read(inStream, hdr.Partitions.First().DiscOffset - srcPos);
srcPos += updateRemovedFiller.Size;
WiiPartitionInfo firstPart = WiiDiscHeaderSection.CreatePartitionInfos(updateRemovedFiller, 0)?.FirstOrDefault(a => a.Type != PartitionType.Update);
string updateFileName = RecoveryData.GetUpdatePartition(ctx.Settings, updatePartitionCrc);
if (updateFileName != null)
{
using (FileStream pf = File.OpenRead(updateFileName))
{
pf.Copy(crcStream, pf.Length);
dstPos += pf.Length;
}
}
else
{
string msg = string.Format("!! Update partition *_{0} missing - Adding filler. It may be Recoverable", updatePartitionCrc.ToString("X8"));
isRecoverable = true;
_log?.LogDetail(msg);
//throw pc.SetReaderException(new HandledException("Failed to convert: " + msg));
}
ByteStream.Zeros.Copy(crcStream, firstPart.DiscOffset - dstPos); //fill full gap
dstPos += firstPart.DiscOffset - dstPos;
}
NkitPartitionPatchInfo patchInfo = new NkitPartitionPatchInfo()
{
HashGroups = new Dictionary <long, MemorySection>()
};
patchInfos.Add(patchInfo);
if (part.DiscOffset > srcPos)
{
dstPos += writeFiller(ref srcPos, dstPos, dstPos + 0x1cL, inStream, crcStream, new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize), scrubFiller);
inStream.Copy(ByteStream.Zeros, part.DiscOffset - srcPos); //padded to 0x8000
srcPos += part.DiscOffset - srcPos;
}
part.DiscOffset = dstPos; //restore the original position
patchInfo.DiscOffset = dstPos;
patchInfo.PartitionHeader = MemorySection.Read(inStream, 0x20000);
srcPos += patchInfo.PartitionHeader.Size;
long size = patchInfo.PartitionHeader.ReadUInt32B(0x2bc) * 4L;
LongRef origSize = new LongRef()
{
Value = 0
};
WiiPartitionGroupSection wp = null;
WiiPartitionHeaderSection wh = new WiiPartitionHeaderSection(hdr, null, part.DiscOffset, patchInfo.PartitionHeader.Data, patchInfo.PartitionHeader.Data.Length);
MemorySection ph = new MemorySection(new byte[0x8000 * 64]);
long remaining = long.MaxValue;//set after first block read
int groupIndex = 0;
WiiHashStore hashes = new WiiHashStore();
patchInfo.ScrubManager = wh.ScrubManager;
bool patchBlock = false;
StreamCircularBuffer prtStream = null;
try
{
using (prtStream = new StreamCircularBuffer(0, null, null, output => srcPos += partitionStreamWrite(origSize, inStream, output, size, ctx.Dats, patchInfo, hashes, pc)))
{
int gs = 0;
int ge = 0;
int i = 0;
MemoryStream patchBlocks = null;
while (remaining > 0)
{
int blocks = (int)Math.Min(64L, remaining / 0x7c00);
for (int b = 0; b < blocks; b++)
{
prtStream.Read(ph.Data, (b * 0x8000) + 0x400, 0x7c00); //load aligned with no hashes
if (remaining == long.MaxValue) //first loop
{
remaining = origSize.Value;
if (ph.ReadString(0x400 + 0, 4) == "\0\0\0\0")
{
gs = -1;
ge = -1;
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, (uint)(NStream.DataToHashedLen(origSize.Value) / 4)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
}
else
{
gs = (int)((ph.ReadUInt32B(0x400 + 0x424) * 4L) / (0x7c00L * 64));
ge = (int)(((ph.ReadUInt32B(0x400 + 0x424) * 4L) + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) / (0x7c00L * 64));
if ((int)((part.DiscOffset + (ph.ReadUInt32B(0x400 + 0x428) * 4L)) % (0x7c00L * 64)) == 0)
{
ge--; //don't load the next group if the data will end on the last byte
}
blocks = (int)Math.Min(64L, remaining / 0x7c00);
lastPartitionId = ph.ReadString(0x400 + 0, 4);
patchInfo.PartitionHeader.WriteUInt32B(0x2bc, ph.ReadUInt32B(0x400 + 0x210)); //restore real size
crcStream.Write(patchInfo.PartitionHeader.Data, 0, patchInfo.PartitionHeader.Data.Length);
dstPos += patchInfo.PartitionHeader.Size;
ph.WriteUInt32B(0x400 + 0x200, 0);
ph.WriteUInt32B(0x400 + 0x204, 0);
ph.WriteUInt32B(0x400 + 0x208, 0);
ph.WriteUInt32B(0x400 + 0x20C, 0);
ph.WriteUInt32B(0x400 + 0x210, 0);
ph.WriteUInt32B(0x400 + 0x214, 0);
ph.WriteUInt32B(0x400 + 0x218, 0);
}
wp = new WiiPartitionGroupSection(hdr, wh, ph.Data, part.DiscOffset, blocks * 0x8000, false);
wh.Initialise(wp, origSize.Value);
}
}
if (blocks < 64)
{
Array.Clear(ph.Data, blocks * 0x8000, ph.Data.Length - (blocks * 0x8000)); //clear remaining blocks
}
wp.Populate(groupIndex, ph.Data, dstPos, blocks * 0x8000);
int scrubbed = 0;
for (int bi = 0; bi < blocks; bi++)
{
wp.MarkBlockDirty(bi);
byte byt;
if (patchInfo.ScrubManager.IsBlockScrubbedScanMode(wp.Offset + (bi * 0x8000), out byt))
{
wp.SetScrubbed(bi, byt);
scrubbed++;
}
}
bool isFstBlocks = i >= gs && i <= ge;
bool reqHashes = hashes.IsPreserved(wp.Offset); //test with 0 partition based offset
repairBlocks(wp, scrubbed, blocks, false, isFstBlocks); //only test if the hashes aren't preserved (only preserved for scrubbed/customs)
if (reqHashes) //store with disc based offset
{
patchInfo.HashGroups.Add(wp.Offset + part.DiscOffset + patchInfo.PartitionHeader.Size, new MemorySection((byte[])wp.Decrypted.Clone())); //fetch the stored hashed that couldn't be recreated
}
groupIndex++;
bool inFstArea = i >= gs && i <= ge;
if (!patchBlock && (gs == i || reqHashes))
{
patchBlocks = new MemoryStream();
crc.Snapshot(lastPartitionId + " Data");
patchBlock = true;
}
else if (patchBlock && !inFstArea && !reqHashes)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
patchBlock = false;
}
#if DECRYPT
outStream.Write(wp.Decrypted, 0, blocks * 0x8000);
if (i >= gs && i <= ge)
{
fstBlocks.Write(wp.Decrypted, 0, blocks * 0x8000);
}
#else
crcStream.Write(wp.Encrypted, 0, blocks * 0x8000);
if (patchBlock)
{
patchBlocks.Write(wp.Encrypted, 0, blocks * 0x8000);
}
#endif
remaining -= (blocks * 0x7c00);
dstPos += (blocks * 0x8000);
i++;
}
if (patchBlock)
{
crc.Snapshot(lastPartitionId + " Patch");
crc.Crcs.Last().PatchData = patchBlocks.ToArray();
patchBlocks.Dispose();
}
}
if (origSize.Value != prtStream.WriterPosition)
{
throw pc.SetReaderException(new HandledException("Partition read did not write the full amount to the circular buffer before completing"));
}
}
catch (Exception ex)
{
if (prtStream?.WriterException != null)
{
throw pc.SetReaderException(prtStream.WriterException, "Failed reading filesystem");
}
throw pc.SetReaderException(ex, "Failed converting the filesystem");; //writer exception
}
srcPos += hashes.ReadPatchData(part.DiscOffset + patchInfo.PartitionHeader.Size, patchInfo.HashGroups, inStream);
//read hash patches
lastPartitionType = part.Type;
}
if (srcPos < inStream.Length)
{
JunkStream partJunk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
dstPos += writeFiller(ref srcPos, dstPos, lastPartitionType == PartitionType.Update ? imageSize : dstPos + 0x1cL, inStream, crcStream, partJunk, scrubFiller);
}
}
crc.Snapshot("End");
if (updatePartitionCrc != 0)
{
hdr.Write((int)WiiDiscHeaderSection.PartitionTableOffset, updateRemovedFiller.Data, 0, (int)WiiDiscHeaderSection.PartitionTableLength); //restore the exact partition table if update was removed
}
else
{
hdr.UpdateOffsets(); //just update the table with the new offsets
}
crc.Crcs[0].PatchData = hdr.Data;
foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
{
NkitPartitionPatchInfo patchInfo = patchInfos.FirstOrDefault(a => ci.Offset >= a.DiscOffset + a.PartitionHeader.Size && ci.Offset < a.DiscOffset + a.PartitionHeader.Size + a.Size);
if (patchInfo != null)
{
patchGroups(patchInfo, hdr, ci.Offset, ci.PatchData);
}
ci.PatchCrc = Crc.Compute(ci.PatchData);
}
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, isRecoverable, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - Read and convert");
}
}
