private long copyFile(ConvertFile file, ref long nullsPos, ref long srcPos, long dstPos, Stream inStream, Stream target)
{
FstFile ff = file.FstFile;
long size = ff.Length;
if (size == 0)
{
return(0); //could be legit or junk
}
size += size % 4 == 0 ? 0 : 4 - (size % 4);
try
{
inStream.Copy(target, size);
}
catch (Exception ex)
{
throw new HandledException(ex, "Copy file '{0}' failed at Data Position {1} ({2} bytes)", ff.Name, ff.DataOffset.ToString("X"), ff.Length.ToString());
}
srcPos += size;
dstPos += size;
nullsPos = dstPos + 0x1CL;
return(size);
}
public static FileSystem Parse(Stream fstData, long fstOffset, long length, string id, bool isGc)
{
MemorySection ms = MemorySection.Read(fstData, length);
FstFile ff = new FstFile(null)
{
Name = "fst.bin", DataOffset = fstOffset, Offset = NStream.DataToOffset(fstOffset, !isGc), IsNonFstFile = true, Length = (int)fstData.Length
};
return(Parse(ms, ff, id, isGc));
}
internal void ParseFst(byte[] fst)
{
if (_fileSystem == null && _fst == null)
{
_fst = fst;
}
if (this.FileSystem == null)
{
return;
}
_fileSystem = this.FileSystem; //force read of filesystem here
List <Tuple <long, int, FstFile> > h3 = new List <Tuple <long, int, FstFile> >();
List <FstFile> nonNull = _fileSystem.Files.Where(a => a.Length != 0).ToList();
for (int i = 0; i < nonNull.Count; i++) //already ordered by offset
{
FstFile fl = nonNull[i];
FstFile fn = null;
if (i + 1 < nonNull.Count)
{
fn = nonNull[i + 1];
}
long end = fl.DataOffset + fl.Length;
if (end % 0x7c00L == 0 || (int)(end % 0x7c00L) > 0x7c00 - 28)
{
if (fn == null)
{
h3.Add(new Tuple <long, int, FstFile>(fl.DataOffset, 28, fl));
}
else
{
int nullCount = 28;
if (end % 0x7c00L != 0)
{
nullCount = (0x7c00 - (int)(end % 0x7c00L));
end += nullCount;
nullCount = 28 - nullCount;
}
long diff = (int)(fn.DataOffset - end);
if (diff >= 0x7c00)
{
h3.Add(new Tuple <long, int, FstFile>(end, nullCount + (nullCount % 2), fl)); //data offset
}
}
}
}
_scrubManager.H3Nulls.AddRange(h3.Union(_fileSystem.Files.Where(a => a.Length == 0).Select(a => new Tuple <long, int, FstFile>(a.DataOffset, 28, a))));
}
internal static FileSystem Parse(MemorySection ms, FstFile fst, string id, bool isGc)
{
FstFolder fld = new FstFolder(null);
long nFiles = ms.ReadUInt32B(0x8);
if (12 * nFiles > ms.Size)
{
return(null);
}
if (fst != null)
{
fld.Files.Add(fst);
}
recurseFst(ms, fld, 12 * nFiles, 0, id, isGc);
return(new FileSystem(fld));
}
private long copyFile(ConvertFile file, ref long nullsPos, ref long srcPos, long dstPos, long imageSize, Stream inStream, Stream target)
{
FstFile ff = file.FstFile;
long size = ff.Length;
if (size == 0)
{
return(0); //could be legit or junk
}
size += size % 4 == 0 ? 0 : 4 - (size % 4);
size = Math.Min(size, imageSize - dstPos); //never overwrite file length (starfox e3 fix)
inStream.Copy(target, size);
srcPos += size;
dstPos += size;
nullsPos = dstPos + 0x1CL;
return(size);
}
private bool areaUsed(FstFile file, long offset, long size)
{
bool b = (file.DataOffset > offset && file.DataOffset < offset + size); //starts within range
if (b)
{
return(true);
}
b = (file.DataOffset + file.Length > offset && file.DataOffset + file.Length < offset + size); //ends within range
if (b)
{
return(true);
}
b = (file.DataOffset <offset && file.DataOffset + file.Length> offset + size);
if (b)
{
return(true);
}
return(false);
}
private long writeFile(ref long nullsPos, Stream dest, long dstPos, ref bool firstFile, Stream srcStream, FstFile lastFile, FstFile file, List <JunkRedumpPatch> patches, NStream nstream, long junkStart, bool isJunkFile)
{
//Debug.WriteLine(string.Format(@"{0} : {1} : {2} : {3}/{4}", file.DataOffset.ToString("X8"), (file.DataOffset + file.Length).ToString("X8"), /*(nextFile.DataOffset - lastEnd).ToString("X8"),*/ file.Length.ToString("X8"), file.Path, file.Name));
//file found
long gap = writeDestGap(nullsPos, dest, dstPos, file.DataOffset - dstPos, firstFile, patches, junkStart, nstream);
firstFile = false;
bool missing = false;
if (file.Length == 0)
{
// missing = true;
}
else if (isJunkFile)
{
missing = true;
MemorySection ms = MemorySection.Read(srcStream, 8);
long size = ms.ReadUInt32B(0);
GapType gt = (GapType)(size & 0b11);
size &= 0xFFFFFFFC;
//set nullsPos value if zerobyte file without junk
if (gt == GapType.JunkFile)
{
nullsPos = Math.Min(nullsPos - (dstPos + gap), 0); //reset the nulls
long nulls = (size & 0xFC) >> 2;
long junkFileLen = ms.ReadUInt32B(4);
if (junkFileLen != file.Length)
{
throw new HandledException(string.Format("NKit Junk file restoration length mismatch {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
ByteStream.Zeros.Copy(dest, nulls);
nstream.JunkStream.Position = (dstPos + gap) + nulls;
nstream.JunkStream.Copy(dest, junkFileLen - nulls);
_log?.LogDetail(string.Format("Generated file content with Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
else
{
throw new HandledException(string.Format("NKit Junk file restoration bytes invalid {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
}
else
{
byte[] f = new byte[Math.Min(0x30, file.Length)];
srcStream.Read(f, 0, f.Length); //then read while junk is created
if (lastFile.DataOffset == file.DataOffset && lastFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size
{
nullsPos = dstPos + 0x1CL; //will already be aligned
}
int nulls = (int)(nullsPos - (dstPos + gap));
nstream.JunkStream.Position = file.DataOffset; //async junk gen
int countNulls = 0;
for (int i = 0; i < f.Length && f[i] == 0; i++)
{
countNulls++;
}
if (f.Length > nulls && countNulls < f.Length) //don't test all nulls
{
missing = nstream.JunkStream.Compare(f, 0, f.Length, Math.Max(0, nulls)) == f.Length;
}
if (missing)
{
_log?.LogDetail(string.Format("File content is Junk {0}: {1}", file.DataOffset.ToString("X8"), file.Name));
}
dest.Write(f, 0, f.Length);
srcStream.Copy(dest, file.Length - f.Length); //copy file
}
if (!missing) //reset the gap when no junk
{
nullsPos = dstPos + gap + file.Length + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
}
return(gap + file.Length);
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
string resultMsg = "";
if (!Settings.ConfigFileFound)
{
_log?.Log("!! No config file found - This is required to restore and validate images");
}
NCrc crc = new NCrc();
Settings settings = ctx.Settings;
List <string> addedFiles = new List <string>();
DatData data = ctx.Dats;
RecoveryData rec = ctx.Recovery;
List <JunkRedumpPatch> patches = rec.JunkPatches;
MemorySection hdr = null;
try
{
long junkStart = settings.JunkStartOffsets.FirstOrDefault(a => a.Id8 == inStream.Id8)?.Offset ?? 0;
string forceJunkId = settings.JunkIdSubstitutions.FirstOrDefault(a => a.Id8 == inStream.Id8)?.JunkId;
if (forceJunkId != null)
{
_log?.LogDetail(string.Format("Using ID {0} for junk not image ID {1}", forceJunkId, inStream.Id));
//result.ImageInfo.JunkId = forceJunkId;
inStream.ChangeJunk(forceJunkId);
}
if (junkStart != 0)
{
_log?.LogDetail(string.Format("Junk forced to start at 0x{0}", junkStart.ToString("X8")));
}
hdr = inStream.DiscHeader;
FstFileItem goodFst = rec.GcBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8 && !(inStream.Id8 == "GNHE5d0000" && a.Length == 116));
if (goodFst != null)
{
_log?.LogDetail(string.Format("Recovery: {0}", goodFst.Filename));
}
if (goodFst == null && rec.GcNewBinFiles != null)
{
goodFst = rec.GcNewBinFiles.Where(a => a is FstFileItem).Cast <FstFileItem>().FirstOrDefault(a => a.Id8 == inStream.Id8);
}
if (goodFst != null)
{
goodFst.Populate();
}
ApploaderFileItem goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
if (goodAldr == null && rec.GcNewBinFiles != null)
{
goodAldr = goodFst == null ? null : (ApploaderFileItem)rec.GcNewBinFiles.FirstOrDefault(a => a.Crc == goodFst.AppLoadCrc);
}
//is it an action replay (custom hacks)
if ((hdr.ReadUInt32B(0x420) == 0 && inStream.Id8 == "GNHE5d0000") || inStream.Id8 == "DTLX010000" || inStream.Id8 == "102E010007")
{
if (inStream.Id8 == "102E010007")
{
resultMsg = "Aging Disc detected - Skipping recover";
}
else
{
resultMsg = "Datel Action Replay detected - Skipping recover";
}
try
{
using (CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write))
{
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
inStream.Copy(target, pc.OutputSize - hdr.Size);
}
_log?.LogDetail(resultMsg);
}
catch (Exception ex)
{
throw new HandledException(ex, resultMsg);
}
crc.Snapshot("files");
}
else
{
//############################################################################
//# READ DISC START
//########### Header (boot.bin) 0 (read by base stream already)
long srcPos = hdr.Size;
_log?.LogDetail(string.Format("Header Read: {0} - {1}", friendly(inStream.Id), friendly(inStream.Title)));
//########### Header Info (bi2.bin) 0x440
MemorySection bi2bin = MemorySection.Read(inStream, 0x2000);
srcPos += bi2bin.Size;
//########### APPLOADER (appldr.bin) 0x2440 - Action Reply can have 0 as the main.dol
MemorySection appldr = MemorySection.Read(inStream, Math.Min(hdr.ReadUInt32B(0x420) == 0 ? uint.MaxValue : hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - srcPos);
srcPos += appldr.Size;
//########### APP (main.dol)
MemorySection maindol;
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424))
{
maindol = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - srcPos);
}
else
{
maindol = new MemorySection(new byte[0]);
}
srcPos += maindol.Size;
//########### FST (fst.bin)
MemorySection srcFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428));
srcPos += hdr.ReadUInt32B(0x428);
//############################################################################
//# CORRECT ISSUES
List <FstFile> srcFiles = FileSystem.Parse(srcFst, null, inStream.Id, true)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
if (srcFiles == null)
{
throw new HandledException(string.Format("FST Corrupt or misaligned, could not be parsed at position 0x{0}", hdr.ReadUInt32B(0x424).ToString("X8")));
}
uint crcTmp = Crc.Compute(appldr.Data, 0, Math.Min((int)appldr.Size, 0x20 + (int)(appldr.ReadUInt32B(0x14) + appldr.ReadUInt32B(0x18))));
//adjust appldr
if (goodAldr != null && crcTmp != goodAldr.Crc)
{
_log?.LogDetail(string.Format("Replacing appldr.bin crc {0} with Recovery appldr.bin {1}", crcTmp.ToString("X8"), goodAldr.Crc.ToString("X8")));
addedFiles.Add(goodAldr.Filename);
appldr = new MemorySection(File.ReadAllBytes(goodAldr.Filename));
}
//adjust main.dol
if (goodFst != null)
{
if (goodFst.MainDolOffset != hdr.ReadUInt32B(0x420))
{
//main.dol is before fst in image and after in goodfst
if (hdr.ReadUInt32B(0x420) < hdr.ReadUInt32B(0x424) && goodFst.MainDolOffset > goodFst.FstOffset)
{
_log?.LogDetail(string.Format("Skipping main.dol at 0x{0} using 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
maindol = new MemorySection(new byte[0]);
}
else
{
_log?.LogDetail(string.Format("Moving main.dol address 0x{0} to 0x{1}", hdr.ReadUInt32B(0x420).ToString("X8"), goodFst.MainDolOffset.ToString("X8")));
}
hdr.WriteUInt32B(0x420, (uint)goodFst.MainDolOffset);
}
if (goodFst.Region != (Region)bi2bin.ReadUInt32B(0x18))
{
_log?.LogDetail(string.Format("Region Changed to {0} from {1}", goodFst.Region.ToString(), ((Region)bi2bin.ReadUInt32B(0x18)).ToString()));
bi2bin.WriteUInt32B(0x18, (uint)goodFst.Region);
}
if (goodFst.MaxFst != hdr.ReadUInt32B(0x42C))
{
_log?.LogDetail(string.Format("Max Fst Size changed to {0} from {1}", goodFst.MaxFst.ToString("X8"), hdr.ReadUInt32B(0x42C).ToString("X8")));
hdr.WriteUInt32B(0x42C, (uint)goodFst.MaxFst);
}
string newTitle = Encoding.ASCII.GetString(goodFst.Title);
if (newTitle != hdr.ReadString(0x20, 0x60 - 0x20))
{
_log?.LogDetail(string.Format("Title changed to '{0}' from '{1}'", newTitle.TrimEnd('\0'), hdr.ReadString(0x20, 0x60 - 0x20).TrimEnd('\0')));
hdr.Write(0x20, goodFst.Title);
}
}
MemorySection fst = srcFst;
List <FstFile> fstFiles = srcFiles;
crcTmp = Crc.Compute(srcFst.Data);
if (goodFst != null && crcTmp != goodFst.Crc)
{
_log?.LogDetail(string.Format("Replacing fst.bin crc {0} with Recovery fst {1}", crcTmp.ToString("X8"), goodFst.Crc.ToString("X8")));
addedFiles.Add(goodFst.Filename);
fst = new MemorySection(goodFst.FstData);
fstFiles = FileSystem.Parse(fst, null, inStream.Id, true).Files.OrderBy(a => a.Offset).ThenBy(a => a.Length).ToList();
}
//adjust fst.bin
if (goodFst != null && (goodFst.FstOffset != hdr.ReadUInt32B(0x424) || goodFst.FstData.Length != hdr.ReadUInt32B(0x428)))
{
_log?.LogDetail(string.Format("Moving / Resizing fst.bin address 0x{0} (Length {1}) to 0x{2} (Length {3})", hdr.ReadUInt32B(0x424).ToString("X8"), hdr.ReadUInt32B(0x428).ToString(), goodFst.FstOffset.ToString("X8"), goodFst.FstData.Length.ToString()));
hdr.WriteUInt32B(0x424, (uint)goodFst.FstOffset);
hdr.WriteUInt32B(0x428, (uint)goodFst.FstData.Length);
}
int c;
int failCount = 0;
int filesMoved = 0;
int nkitJunkFiles = 0;
foreach (FstFile f in srcFiles)
{
FstFile[] fnd = fstFiles.Where(a => a.Name == f.Name && a.Path == f.Path).ToArray();
if (fnd.Length == 0)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: No File Found - {0}/{1} (Length {2})", f.Path, f.Name, f.Length.ToString()));
}
else if (inStream.IsNkit && (c = fnd.Count(a => a.Length != 0 && f.Length == 0)) == 1)
{
nkitJunkFiles++;
}
else if ((c = fnd.Count(a => a.Length == f.Length)) != 1)
{
failCount++;
_log?.LogDetail(string.Format("FST Error: File Size bad - {0}/{1} (Length {2}){3}", f.Path, f.Name, f.Length.ToString(), c <= 1 ? "" : string.Format(" {0} files found", c.ToString())));
}
else if (f.DataOffset != fnd[0].DataOffset)
{
filesMoved++;
}
if (failCount >= 10)
{
break;
}
}
if (failCount != 0)
{
if (failCount >= 10)
{
throw new HandledException(string.Format("{0} or more FST errors found", failCount.ToString()));
}
}
if (filesMoved != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be repositioned when rebuilding this image", filesMoved.ToString(), filesMoved == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (nkitJunkFiles != 0)
{
_log?.LogDetail(string.Format("{0} file{1} of {2} will be generated from junk when rebuilding this image", nkitJunkFiles.ToString(), nkitJunkFiles == 1 ? "" : "s", fstFiles.Count.ToString()));
}
if (goodFst != null)
{
//is the data output so far correct
if (!bruteForceValidHeader(ctx, goodFst, hdr, bi2bin, appldr, maindol, fst, pc))
{
throw new HandledException(string.Format("Post FST Crc Failed 0x{0}", goodFst.PostFstCrc.ToString("X8")));
}
}
//############################################################################
//# WRITE DISC START
CryptoStream target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(forceJunkId, 0); //size that we will output from this read
long dstPos = hdr.Data.Length;
crc.Snapshot("boot.bin");
target.Write(bi2bin.Data, 0, (int)bi2bin.Size);
dstPos += bi2bin.Size;
crc.Snapshot("bi2.bin");
target.Write(appldr.Data, 0, (int)appldr.Size);
dstPos += appldr.Size;
ByteStream.Zeros.Copy(target, Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos);
dstPos += Math.Min(hdr.ReadUInt32B(0x420), hdr.ReadUInt32B(0x424)) - dstPos;
crc.Snapshot("appldr.bin");
target.Write(maindol.Data, 0, (int)maindol.Size);
dstPos += maindol.Size;
if (goodFst != null)
{
long padding = goodFst.FstOffset - dstPos;
ByteStream.Zeros.Copy(target, padding);
dstPos += padding;
}
crc.Snapshot("main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
dstPos += fst.Size;
bool firstFile = true;
Dictionary <FstFile, byte[]> cache = new Dictionary <FstFile, byte[]>();
crc.Snapshot("fst.bin");
//############################################################################
//# WRITE THE FILESYSTEM
int fidx = -1;
FstFile lastFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = fst.Size
};
FstFile nextFile = fstFiles[++fidx];
FstFile cacheFile = null;
long nullsPos = dstPos + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
//########### FILES
foreach (FstFile f in srcFiles) //read the files and write them out as goodFiles (possible order difference)
{
while ((cacheFile = cache.Keys.FirstOrDefault(a => (a.Length == nextFile.Length || (inStream.IsNkit && a.Length == 0)) && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null) //write cache
{
bool isJunk = inStream.IsNkit && cacheFile.Length == 0 && cacheFile.Name == nextFile.Name && cacheFile.Path == nextFile.Path;
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, isJunk);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
if (f.DataOffset - srcPos > 0) //skip src junk
{
inStream.Copy(ByteStream.Zeros, f.DataOffset - srcPos);
srcPos += f.DataOffset - srcPos;
}
if (nextFile != null && (f.Length != nextFile.Length || f.Name != nextFile.Name || f.Path != nextFile.Path)) //cache file (nkit junk files are cached)
{
bool isNkitJunk = inStream.IsNkit && f.Length == 0 && nextFile.Length != 0 && f.Name == nextFile.Name && f.Path == nextFile.Path;
byte[] cacheItem = new byte[isNkitJunk ? 8 : f.Length];
inStream.Read(cacheItem, 0, cacheItem.Length); //read the nkit junk data - details (real length and null counts)
cache.Add(f, cacheItem);
srcPos += cacheItem.Length;
}
else //copy file
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, inStream, lastFile, nextFile, patches, inStream, junkStart, false);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
srcPos += f.Length;
}
}
while (nextFile != null && fidx < fstFiles.Count && (cacheFile = cache.Keys.FirstOrDefault(a => a.Length == nextFile.Length && a.Name == nextFile.Name && a.Path == nextFile.Path)) != null)
{
using (MemoryStream cacheStream = new MemoryStream(cache[cacheFile]))
{
dstPos += writeFile(ref nullsPos, target, dstPos, ref firstFile, cacheStream, lastFile, nextFile, patches, inStream, junkStart, false);
}
cache.Remove(cacheFile);
lastFile = nextFile;
nextFile = fidx + 1 < fstFiles.Count ? fstFiles[++fidx] : null;
}
writeDestGap(nullsPos, target, dstPos, pc.OutputSize - dstPos, true, patches, junkStart, inStream);
crc.Snapshot("files");
}
resultMsg = "MatchFail";
uint finalCrc = crc.FullCrc(true);
if (ctx.Dats.RedumpData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Redump";
}
if (ctx.Dats.CustomData.FirstOrDefault(a => a.Crc == finalCrc) != null)
{
resultMsg = "Match Custom";
}
pc.ReaderCheckPoint2Complete(crc, false, finalCrc, finalCrc, VerifyIsWrite, hdr.Data, resultMsg);
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "RestoreReaderGc.Read - Read and repair"); //don't let the writer lock
}
}
public void SeekToFile(FstFile file)
{
_seek = file.Offset - (file.Offset % GroupSize); //offset within partition of group, set to group boundary
}
public override string ToString()
{
return(string.Format("{0} : {1} : {2}", FstFile.ToString(), GapLength.ToString("X8"), Alignment.ToString()));
}
internal static FileSystem Parse(MemorySection ms, FstFile fst, string id)
{
return(Parse(ms, fst, id, false));
}
internal static long CopyFile(ref long nullsPos, ConvertFile conFile, FstFile prevFile, Stream dest, ref long srcPos, long dstPos, Stream srcStream, JunkStream junkNStream, long imageSize, out bool missing)
{
//long pos = dest.Position;
missing = false;
FstFile file = conFile.FstFile;
long size = file.Length + (file.Length % 4 == 0 ? 0 : (4 - (file.Length % 4)));
if (srcPos + size > imageSize)
{
size = file.Length; //some rare GC demos end at the end of a non aligned file. This fixes it - v1.2 bugfix
}
long written = size;
byte[] f = new byte[Math.Min(0x30, size)];
srcStream.Read(f, 0, f.Length); //then read while junk is created
if (prevFile != null && prevFile.DataOffset == file.DataOffset && prevFile.Length == 0) //null file overlapped this file so set nullsPos to have a gap (XGIII) needs fst sorting by offset then size
{
nullsPos = srcPos + 0x1CL; //will already be aligned
}
int nulls = (int)(nullsPos - srcPos);
int countNulls = 0;
if (f.Length > nulls)
{
junkNStream.Position = file.DataOffset; //async junk gen
for (int i = 0; i < f.Length && f[i] == 0; i++)
{
countNulls++;
}
if (countNulls < f.Length) //don't test all nulls
{
missing = junkNStream.Compare(f, 0, f.Length, Math.Max(0, countNulls)) == f.Length;
}
}
if (missing) //start of file is junk
{
//check the remainder of the file
MemorySection junkFile = MemorySection.Read(srcStream, size - f.Length);
missing = junkNStream.Compare(junkFile.Data, 0, (int)junkFile.Size, 0) == junkFile.Size;
if (missing)
{
written = 0;
conFile.Gap.SetJunkFile((uint)conFile.FstFile.Length, countNulls);
}
else //not 100% junk, write the file out
{
dest.Write(f, 0, f.Length);
dest.Write(junkFile.Data, 0, (int)junkFile.Size);
}
junkFile = null;
}
else
{
dest.Write(f, 0, f.Length);
srcStream.Copy(dest, size - f.Length); //copy file
}
if (!missing) //reset the gap when no junk
{
nullsPos = srcPos + size + 0x1c;
if (nullsPos % 4 != 0)
{
nullsPos += 4 - (nullsPos % 4);
}
}
srcPos += size;
return(written);
}
internal static List <ConvertFile> GetConvertFstFiles(Stream inStream, long size, MemorySection hdr, MemorySection fst, bool isGc, long fstFileAlignment, out string error)
{
string[] align = new string[] { ".tgc" }; //, ".dol", ".thp", ".adp", ".poo", ".pcm", ".pcm16", ".son", ".act", ".bin", ".gpl", ".tpl", ".bmp", ".sam", ".sdi", ".tcs", ".txt", ".skn", ".stp", ".anm" };
error = null;
List <ConvertFile> conFiles = new List <ConvertFile>();
try
{
List <FstFile> srcFiles = FileSystem.Parse(fst, null, hdr.ReadString(0, 4), isGc)?.Files?.OrderBy(a => a.Offset)?.ThenBy(a => a.Length)?.ToList();
//get list of files and gaps
long end;
long gap;
long fstLen = hdr.ReadUInt32B(0x424) * (isGc ? 1L : 4L);
FstFile ff = new FstFile(null)
{
Name = "fst.bin", DataOffset = fstLen, Offset = fstLen, Length = (int)fst.Size, IsNonFstFile = true
};
for (int i = 0; i < srcFiles.Count; i++)
{
ff = i == 0 ? ff : srcFiles[i - 1];
end = ff.DataOffset + ff.Length;
end += end % 4 == 0 ? 0 : 4 - (end % 4);
gap = srcFiles[i].DataOffset - end;
if (gap < 0)
{
error = string.Format("The gap between '{0}' and '{1}' is {2} - Converting as bad image", ff.Name, srcFiles[i].Name, gap.ToString());
return(null);
}
conFiles.Add(new ConvertFile(gap, isGc)
{
FstFile = ff
});
}
ff = srcFiles.Last();
end = ff.DataOffset + ff.Length;
end += end % 4 == 0 ? 0 : 4 - (end % 4);
gap = size - end;
if (gap >= -3 && gap < 0)
{
gap = 0; //some hacked gc images converted from tgc end on the file end (star fox e3)
}
if (gap < 0)
{
error = string.Format("The gap between '{0}' and the end of the image is {1} - Converting as bad image/partition", ff.Name, gap.ToString());
return(null);
}
conFiles.Add(new ConvertFile(gap, isGc)
{
FstFile = ff
});
//set alignment
foreach (ConvertFile cf in conFiles)
{
ff = cf.FstFile;
if (fstFileAlignment == 0)
{
cf.Alignment = 0; //preserve alignment
}
else if (fstFileAlignment == -1 && ff.DataOffset % 0x8000 == 0 && (ff.Length % 0x8000 == 0 || align.Contains(Path.GetExtension(ff.Name).ToLower())))
{
cf.Alignment = 0x8000; //default behaviour
}
else if (fstFileAlignment != 0 && ff.DataOffset % fstFileAlignment == 0) //src matches alignment
{
cf.Alignment = fstFileAlignment; //align to largest multiple
}
else
{
cf.Alignment = -1; //-1 = do not align this file
}
}
}
catch
{
error = "Fst parsing error - Converting as bad image";
return(null);
}
return(conFiles);
}
public void Read(Context ctx, NStream inStream, Stream outStream, Coordinator pc)
{
try
{
DatData data = ctx.Dats;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = inStream.DiscHeader;
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
uint nkitCrc = hdr.ReadUInt32B(0x208);
uint imageSize = hdr.ReadUInt32B(0x210);
string junkId = hdr.ReadString(0x214, 4);
if (junkId != "\0\0\0\0")
{
inStream.ChangeJunk(junkId);
}
hdr.WriteUInt32B(0x200, 0);
hdr.WriteUInt32B(0x204, 0);
hdr.WriteUInt32B(0x208, 0);
hdr.WriteUInt32B(0x20C, 0);
hdr.WriteUInt32B(0x210, 0);
hdr.WriteUInt32B(0x214, 0);
hdr.WriteUInt32B(0x218, 0);
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
NCrc crc = new NCrc();
NStream target = new NStream(new CryptoStream(outStream, crc, CryptoStreamMode.Write));
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x424) - hdr.Size);
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) + (hdr.ReadUInt32B(0x428) % 4 == 0 ? 0 : 4 - (hdr.ReadUInt32B(0x428) % 4)));
long srcPos = hdr.ReadUInt32B(0x424) + fst.Size;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
pc.ReaderCheckPoint1PreWrite(junkId, hdr.ReadUInt32B(0x208)); //size that we will output from this read
crc.Snapshot("hdr.bin");
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
crc.Snapshot("bi2.bin, appldr.bin, main.dol");
target.Write(fst.Data, 0, fst.Data.Length);
crc.Snapshot("fst.bin");
hdrToFst = null; //let this be collected if needed
long dstPos = hdr.ReadUInt32B(0x424) + fst.Size;
long nullsPos = dstPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, inStream.Length, hdr, fst, true, -1, out error); //result.ImageInfo.IsoSize
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(inStream.Length - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
dstPos += writeGap(cf, ref nullsPos, ref srcPos, dstPos, inStream, target, true);
}
else
{
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), dstPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (dstPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)dstPos);
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)dstPos);
dstPos += copyFile(f, ref nullsPos, ref srcPos, dstPos, imageSize, inStream, target);
}
if (dstPos < imageSize)
{
dstPos += writeGap(f, ref nullsPos, ref srcPos, dstPos, inStream, target, i == 0 || i == conFiles.Count - 1);
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)ff.Length);
}
}
firstFile = false;
}
}
crc.Snapshot("files");
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
crc.Crcs[2].PatchCrc = Crc.Compute(fst.Data);
crc.Crcs[2].PatchData = fst.Data;
if (imageSize != dstPos)
{
throw pc.SetReaderException(new HandledException("Nkit image read output {0} bytes not the expected {1}!", dstPos.ToString(), imageSize.ToString()));
}
pc.ReaderCheckPoint2Complete(crc, false, nkitCrc, crc.FullCrc(true), this.VerifyIsWrite, hdr.Data, nkitCrc == crc.FullCrc(true) ? "NKit Valid" : "NKit Invalid");
pc.ReaderCheckPoint3Complete();
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderGc.Read - Read and convert"); //don't let the writer lock
}
}
private long partitionStreamWrite(LongRef outSize, Stream inStream, Stream target, long size, DatData settingsData, NkitPartitionPatchInfo patchInfo, WiiHashStore hashes, Coordinator pc)
{
DatData data = settingsData;
List <string> addedFiles = new List <string>();
DateTime dt = DateTime.Now;
MemorySection hdr = MemorySection.Read(inStream, 0x440);
long srcPos = hdr.Size;
long outPos = 0;
long imageSize = 0;
try
{
if (hdr.ReadString(0, 4) == "\0\0\0\0")
{
long nullsPos = 0;
long fileLength = -1;
LongRef gapLength = new LongRef()
{
Value = -1
};
target.Write(hdr.Data, 0, (int)hdr.Size);
MemorySection sz = MemorySection.Read(inStream, 4);
srcPos += 4;
outPos += hdr.Size;
imageSize = sz.ReadUInt32B(0) * 4L;
outSize.Value = NStream.HashedLenToData(imageSize);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), outSize.Value); //SET LENGTH FROM HEADER
outPos += writeGap(ref fileLength, gapLength, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
string idVer = hdr.ReadString(0x200, 8);
if (idVer != "NKIT v01")
{
throw new Exception(string.Format("{0} not supported by this version", idVer));
}
bool isNkit = idVer.StartsWith("NKIT");
imageSize = NStream.HashedLenToData((hdr.ReadUInt32B(0x210) * 4L));
outSize.Value = imageSize;
string junkId = hdr.ReadString(0x214, 4);
JunkStream junk = new JunkStream(hdr.Read(0, 4), hdr.Read8(6), imageSize); //SET LENGTH FROM HEADER
MemorySection fst;
long mainDolAddr = hdr.ReadUInt32B(0x420);
//############################################################################
//# READ DISC START
MemorySection hdrToFst = MemorySection.Read(inStream, (hdr.ReadUInt32B(0x424) * 4L) - hdr.Size);
srcPos += hdrToFst.Size;
fst = MemorySection.Read(inStream, hdr.ReadUInt32B(0x428) * 4L);
long postFstPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
srcPos += fst.Size;
hashes.WriteFlagsData(imageSize, inStream);
srcPos += hashes.FlagsLength;
patchInfo.PartitionDataHeader = hdr;
patchInfo.Fst = fst;
//############################################################################
//# WRITE DISC START
target.Write(hdr.Data, 0, (int)hdr.Size);
target.Write(hdrToFst.Data, 0, (int)hdrToFst.Size); //padded when read
target.Write(fst.Data, 0, fst.Data.Length);
hdrToFst = null; //let this be collected if needed
outPos = (hdr.ReadUInt32B(0x424) * 4L) + fst.Size;
long nullsPos = outPos + 0x1c;
string error;
List <ConvertFile> conFiles = NkitFormat.GetConvertFstFiles(inStream, size, hdr, fst, false, -1, out error);
if (conFiles == null)
{
if (error != null)
{
_log?.LogDetail(error);
}
ConvertFile cf = new ConvertFile(imageSize - srcPos, true) //result.ImageInfo.IsoSize
{
FstFile = new FstFile(null)
{
DataOffset = hdr.ReadUInt32B(0x424), Offset = hdr.ReadUInt32B(0x424), Length = (int)fst.Size
},
};
outPos += writeGap(cf, ref nullsPos, ref srcPos, outPos, inStream, target, junk, true, patchInfo.ScrubManager);
}
else
{
conFiles[0].GapLength -= hashes.FlagsLength; //fix for a few customs (no gap between the fst and the first file on the source image, but the hash mask makes it look like there is)
//########### FILES
bool firstFile = true;
for (int i = 0; i < conFiles.Count; i++) //read the files and write them out as goodFiles (possible order difference
{
ConvertFile f = conFiles[i];
FstFile ff = f.FstFile;
if (!firstFile) //fst already written
{
//Debug.WriteLine(string.Format(@"{0}>{1} : {2}>{3} : {4} : {5}/{6}", ff.DataOffset.ToString("X8"), outPos.ToString("X8"), (ff.DataOffset + ff.Length).ToString("X8"), (outPos + ff.Length).ToString("X8"), ff.Length.ToString("X8"), ff.Path, ff.Name));
if (srcPos < ff.DataOffset) //skip any padding (not written for wii currently)
{
inStream.Copy(ByteStream.Zeros, ff.DataOffset - srcPos); //skip any 32k align padding etc
srcPos += ff.DataOffset - srcPos;
}
//write file
if (ff.DataOffset == mainDolAddr)
{
hdr.WriteUInt32B(0x420, (uint)(outPos / 4L));
}
fst.WriteUInt32B(ff.OffsetInFstFile, (uint)(outPos / 4L));
outPos += copyFile(f, ref nullsPos, ref srcPos, outPos, inStream, target);
}
if (outPos < imageSize)
{
long gapLen = writeGap(f, ref nullsPos, ref srcPos, outPos, inStream, target, junk, i == 0 || i == conFiles.Count - 1, patchInfo.ScrubManager);
outPos += gapLen;
if (!firstFile)
{
fst.WriteUInt32B(ff.OffsetInFstFile + 4, (uint)(ff.Length));
}
}
firstFile = false;
}
}
}
return(srcPos);
}
catch (Exception ex)
{
throw pc.SetReaderException(ex, "NkitReaderWii.Read - partitionRead");
}
}