public RedumpInfo GetRedumpEntry(Settings settings, DatData dats, uint crc)
{
RedumpEntry redump = dats.RedumpData.FirstOrDefault(a => a.Crc == crc);
RedumpInfo output = new RedumpInfo();
output.MatchType = MatchType.MatchFail;
if (redump == null)
{
if ((redump = dats.CustomData.FirstOrDefault(a => a.Crc == crc)) != null)
{
output.MatchType = MatchType.Custom;
}
}
else
{
output.MatchType = MatchType.Redump;
}
if (redump != null)
{
output.MatchName = SourceFiles.RemoveExtension(redump.Name, true);
output.Checksums = new ChecksumsResult()
{
Crc = crc, Md5 = redump.Md5, Sha1 = redump.Md5
};
}
return(output);
}
internal static void LogNkitInfo(NkitInfo imageInfo, ILog log, string id, bool isDisc)
{
string pfx = string.Format("NKit {0} [{1}]", isDisc ? "Disc" : "Prtn", SourceFiles.CleanseFileName(id).PadRight(4));
log?.LogDetail(string.Format("{0}: In [{1,2:#.0} MiB] ({2} bytes), Out [{3,2:#.0} MiB] (bytes {4})", pfx, imageInfo.BytesReadSize / (double)(1024 * 1024), imageInfo.BytesReadSize.ToString(), imageInfo.BytesWriteSize / (double)(1024 * 1024), imageInfo.BytesWriteSize.ToString()));
if (imageInfo.BytesGcz != 0)
{
log?.LogDetail(string.Format("{0}: GCZ Out [{1,2:#.0} MiB] ({2} bytes)", pfx, imageInfo.BytesGcz / (double)(1024 * 1024), imageInfo.BytesGcz.ToString()));
}
if (imageInfo.BytesJunkFiles != 0)
{
log?.LogDetail(string.Format("{0}: Junk Files Removed [{1,2:#.0} MiB] ({2} bytes)", pfx, imageInfo.BytesJunkFiles / (double)(1024 * 1024), imageInfo.BytesJunkFiles.ToString()));
}
if (imageInfo.BytesHashesData != 0 || imageInfo.BytesHashesPreservation != 0)
{
log?.LogDetail(string.Format("{0}: Hashes In [{1,2:#.0} MiB] ({2} bytes), Preserved [{3,2:#.0} MiB] (bytes {4})", pfx, imageInfo.BytesHashesData / (double)(1024 * 1024), imageInfo.BytesHashesData.ToString(), imageInfo.BytesHashesPreservation / (double)(1024 * 1024), imageInfo.BytesHashesPreservation.ToString()));
}
if (imageInfo.BytesPreservationData != 0)
{
log?.LogDetail(string.Format("{0}: Preservation Data [{1,2:#.0} MiB] {2} bytes", pfx, imageInfo.BytesPreservationData / (double)(1024 * 1024), imageInfo.BytesPreservationData.ToString()));
}
if (imageInfo.BytesPreservationDiscPadding != 0)
{
log?.LogDetail(string.Format("{0}: Preservation Padding [{1,2:#.0} MiB] {2} bytes", pfx, imageInfo.BytesPreservationDiscPadding / (double)(1024 * 1024), imageInfo.BytesPreservationDiscPadding.ToString()));
}
if (imageInfo.FilesTotal != 0 || imageInfo.FilesAligned != 0)
{
log?.LogDetail(string.Format("{0}: {1} Total Files, {2} aligning boundary preserved", pfx, imageInfo.FilesTotal.ToString(), imageInfo.FilesAligned.ToString()));
}
}
private void btnSettingsProcess_Click(object sender, EventArgs e)
{
ListViewItem[] items = itemsToProcess().ToArray();
if (_state == State.New) //if not resuming
{
_processFileIndex = 0;
_processFilesCount = items.Length;
}
setPaths(txtSettingsOutputPathBase.Text);
_state = State.Processing;
setScreenState();
process(cboSettingsMode.SelectedIndex, items,
item => //started item
{
Invoke((MethodInvoker) delegate
{
if (lvw.SelectedItems.Count == 0 || lvw.SelectedItems[0] == _processingItem)
{
item.Selected = true; //move the logging on
}
_processingItem = item;
});
},
item => //completed item
{
Invoke((MethodInvoker) delegate
{
setLog(((ProcessFile)item.Tag).Log);
_processFileIndex++;
OutputResults results = ((ProcessFile)item.Tag).Results;
if (results != null)
{
item.SubItems[1].Text = results.DiscType.ToString();
item.SubItems[2].Text = results.ValidateReadResult.ToString();
item.SubItems[3].Text = results.VerifyOutputResult.ToString();
item.SubItems[4].Text = results.OutputCrc.ToString("X8") ?? "";
item.SubItems[5].Text = SourceFiles.CleanseFileName(results.OutputId4 ?? "");
item.SubItems[6].Text = (results.RedumpInfo?.MatchType.ToString() ?? "") + (results.IsRecoverable ? "Recoverable" : "");
item.SubItems[7].Text = (results.OutputSize / (double)(1024 * 1024)).ToString("#.0") + " MiB";
item.SubItems[8].Text = results.RedumpInfo?.MatchName ?? "";
}
else
{
item.SubItems[1].Text = "Error";
}
});
return(_state != State.Stopping); //stop
}).ContinueWith(
t => //finished processing
{
_state = _processFileIndex == _processFilesCount ? State.Complete : State.Stopped;
setScreenState();
});
}
private void lvw_DragDrop(object sender, DragEventArgs e)
{
string[] files = (string[])e.Data.GetData(DataFormats.FileDrop);
foreach (SourceFile sf in SourceFiles.Scan(files, false))
{
addItem(sf);
}
setScreenState();
}
public string CreateOutputFilename(string newExtension)
{
if (!this.IsArchive)
{
return(SourceFiles.ChangeExtension(this.FilePath, false, newExtension.TrimStart('.')));
}
else
{
return(SourceFiles.ChangeExtension(System.IO.Path.Combine(System.IO.Path.GetDirectoryName(this.FilePath), this.Name), false, newExtension.TrimStart('.')));
}
}
private void saveFile(string path, string imageName, ExtractedFile f, Stream s)
{
path = Path.Combine(path, f.DiscType.ToString(), imageName + "_" + f.DiscId8);
if (f.PartitionId != null)
{
path = Path.Combine(path, f.PartitionId);
}
path = Path.Combine(path, SourceFiles.PathFix(f.Path.TrimStart('/')));
Directory.CreateDirectory(path);
using (Stream b = File.OpenWrite(Path.Combine(path, f.Name)))
s.Copy(b, f.Length);
}
private bool moveToOther(string src, string dst)
{
try
{
dst = Path.Combine(dst, Path.GetFileName(src));
File.Move(src, SourceFiles.GetUniqueName(dst));
return(true);
}
catch
{
return(false);
}
}
public string GetFilename(OutputResults results, string mask)
{
try
{
string fileName = SourceFiles.CleanseFileName(SourceFiles.RemoveExtension(results.InputFileName, true));
string titleName = SourceFiles.CleanseFileName(results.OutputTitle) ?? fileName;
string tgdbName = SourceFiles.CleanseFileName(GameTdbData.FirstOrDefault(a => a.Item1 == results.OutputId6)?.Item2) ?? titleName;
string matchName = results.RedumpInfo?.MatchName != null?SourceFiles.CleanseFileName(results.RedumpInfo.MatchName) : (tgdbName ?? fileName);
Dictionary <string, string> values = new Dictionary <string, string>
{
{ "%src", results.InputFileName ?? Path.GetDirectoryName(Assembly.GetEntryAssembly().Location) },
{ "%nmo", fileName },
{ "%nmg", tgdbName },
{ "%nmd", titleName },
{ "%nmm", matchName },
{ "%dno", results.OutputDiscNo == 0 ? "" : string.Format("(Disc {0})", results.OutputDiscNo.ToString()) },
{ "%ver", results.OutputDiscVersion == 0 ? "" : string.Format("(v1.{0})", results.OutputDiscVersion.ToString("D2")) },
{ "%rev", results.OutputDiscVersion == 0 ? "" : string.Format("(Rev {0})", results.OutputDiscVersion.ToString("D2")) },
{ "%crc", results.OutputCrc.ToString("X8") },
{ "%md5", results.OutputMd5 == null ? "" : BitConverter.ToString(results.OutputMd5).Replace("-", "") },
{ "%sha", results.OutputSha1 == null ? "" : BitConverter.ToString(results.OutputSha1).Replace("-", "") },
{ "%id4", SourceFiles.CleanseFileName(results.OutputId4).PadRight(4) },
{ "%id6", SourceFiles.CleanseFileName(results.OutputId6).PadRight(6) },
{ "%id8", SourceFiles.CleanseFileName(results.OutputId8).PadRight(8) }
};
string[] braces = new[] { "%crc", "%md5", "%sha", "%id4", "%id6", "%id8" };
string fn = mask;
foreach (KeyValuePair <string, string> v in values)
{
if (fn.Contains(v.Key))
{
if (string.IsNullOrEmpty(v.Value) || Regex.IsMatch(fn, Regex.Escape(v.Value), RegexOptions.IgnoreCase))
{
fn = Regex.Replace(fn, string.Format(@"[ ._-]?{0}", v.Key), "", RegexOptions.IgnoreCase);
}
else
{
fn = fn.Replace(v.Key, string.Concat(braces.Contains(v.Key) ? "[" : "", v.Value, braces.Contains(v.Key) ? "]" : ""));
}
}
}
return(SourceFiles.GetUniqueName(fn.Replace("%ext", results.OutputFileExt.Trim('.'))));
}
catch (Exception ex)
{
throw new HandledException(ex, "DatData.GetFilename - Replace masks");
}
}
public void AddRedumpEntry(string datFullFilename, string filename, long size, uint crc, byte[] sha1, byte[] md5)
{
string ne = Path.GetFileName(filename);
string n = SourceFiles.RemoveExtension(filename, true);
if (!File.Exists(datFullFilename))
{
string xml = string.Format(@"<?xml version=""1.0"" encoding=""utf-8""?>
<!DOCTYPE datafile PUBLIC ""-//Logiqx//DTD ROM Management Datafile//EN"" ""http://www.logiqx.com/Dats/datafile.dtd""[]>
<datafile>
<header>
<name>Nintendo - Non-Redump Audit Dat</name>
<description>Created by NKit</description>
<category>Games</category>
<version>1</version>
<date>{0}</date>
<author>You</author>
<email>-not specified-</email>
<homepage>-not specified-</homepage>
<url>-not specified-</url>
<comment>-not specified-</comment>
<clrmamepro />
</header>
</datafile>
", DateTime.Now.ToString("yyyyMMdd"));
Directory.CreateDirectory(Path.GetDirectoryName(datFullFilename));
File.WriteAllText(datFullFilename, xml);
}
if (File.Exists(datFullFilename))
{
XDocument matchDoc = XDocument.Load(datFullFilename);
XmlNamespaceManager namespaceManager = new XmlNamespaceManager(new NameTable());
namespaceManager.AddNamespace("empty", "http://demo.com/2011/demo-schema");
XElement machine = new XElement("machine", new XAttribute("name", n));
matchDoc.Root.Add(machine);
XElement desc = new XElement("description", n);
machine.Add(desc);
XElement rom = new XElement("rom",
new XAttribute("name", ne),
new XAttribute("size", size.ToString()),
new XAttribute("crc", crc.ToString("x8")),
new XAttribute("md5", md5 == null ? "00000000000000000000000000000000" : BitConverter.ToString(md5).Replace("-", "")),
new XAttribute("sha1", md5 == null ? "0000000000000000000000000000000000000000" : BitConverter.ToString(md5).Replace("-", "")));
machine.Add(rom);
matchDoc.Save(datFullFilename);
}
}
public void Write(Context ctx, Stream inStream, Stream outStream, Coordinator pc)
{
try
{
WiiDiscHeaderSection hdr = null;
WiiPartitionHeaderSection pHdr = null;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
NCrc crc = new NCrc();
Crc updateCrc = new Crc();
bool updateRemoved = false;
string updateTmpFileName = null;
string updateFileName = null;
bool extractingUpdate = false;
CryptoStream updateCrcStream = null;
NStream updateTarget = null;
CryptoStream target = null;
MemorySection removedUpdateFiller = null;
int preservedHashCount = 0;
NkitInfo nkitDiscInfo = new NkitInfo();
long fstFileAlignment = -1;
WiiPartitionSection lastPart = null;
long dstPos = 0;
long imageSize = pc.OutputSize; //for Wii: pHdr.PartitionDataLength
string ignoreJunkId;
pc.WriterCheckPoint1WriteReady(out ignoreJunkId); //wait until read has written the header and set the length
NDisc disc = new NDisc(_log, inStream);
foreach (IWiiDiscSection s in disc.EnumerateSections(imageSize)) //no fixing, image should be good //ctx.ImageLength
{
if (s is WiiDiscHeaderSection)
{
hdr = (WiiDiscHeaderSection)s;
hdr.Write8(0x60, 1);
hdr.Write8(0x61, 1);
fstFileAlignment = ctx?.Settings?.PreserveFstFileAlignment?.FirstOrDefault(a => a.Item1 == hdr.Id8)?.Item2 ?? -1;
target = new CryptoStream(outStream, crc, CryptoStreamMode.Write);
target.Write(hdr.Data, 0, hdr.Data.Length); //write the header
nkitDiscInfo.BytesData += hdr.Size;
dstPos += hdr.Size;
crc.Snapshot("Disc Header");
}
else if (s is WiiPartitionSection)
{
WiiPartitionSection ps = (WiiPartitionSection)s;
WiiHashStore hashes = new WiiHashStore(ps.PartitionDataLength);
ScrubManager scrub = ps.Header.ScrubManager;
NkitInfo nkitPartInfo = new NkitInfo();
if (ps.Header.Type == PartitionType.Update && ctx.Settings.NkitUpdatePartitionRemoval && hdr.Partitions.Count() > 1) //only remove update if there's more partitions
{
updateRemoved = true;
extractingUpdate = true;
pHdr = ps.Header;
extractingUpdate = true;
updateCrcStream = new CryptoStream(ByteStream.Zeros, updateCrc, CryptoStreamMode.Write);
crc.Initialize();
nkitPartInfo.BytesData += disc.WriteRecoveryPartitionData(updateCrcStream, false, ps, 0, out updateTmpFileName, out updateFileName, out updateTarget);
_log?.LogDetail(string.Format("Extracted and Removed {0} Recovery Partition: [{1}]", ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));
removedUpdateFiller = new MemorySection(new byte[0x8000]);
removedUpdateFiller.Write(0, hdr.Read(0x40000, 0x100)); //backup the original partition table in case it's non standard in some way
}
else
{
target.Write(ps.Header.Data, 0, (int)ps.Header.Size);
nkitDiscInfo.BytesData += ps.Header.Size;
crc.Snapshot(ps.Id + " Hdr");
crc.Crcs.Last().PatchData = ps.Header.Data;
dstPos += ps.Header.Size;
ps.NewDiscOffset = dstPos;
//long written = 0;
using (StreamCircularBuffer decrypted = new StreamCircularBuffer(ps.PartitionDataLength, null, null, output =>
{
//read decrypted partition
foreach (WiiPartitionGroupSection pg in ps.Sections)
{
nkitPartInfo.BytesHashesData += (pg.Size / 0x8000 * 0x400); //size of input hashes
if (pg.PreserveHashes())
{
nkitPartInfo.BytesHashesPreservation += hashes.Preserve(pg.Offset, pg.Decrypted, pg.Size);
if (++preservedHashCount >= 1500) //too many will bomb the patch caching when converting back. Something is wrong
{
throw pc.SetWriterException(new HandledException("Over 1500 hashes preserved, aborting as image is corrupt or poorly scrubbed."));
}
}
#if !DECRYPT
for (int i = 0; i < pg.Size / 0x8000; i++)
{
output.Write(pg.Decrypted, (i * 0x8000) + 0x400, 0x7c00);
}
#else
output.Write(pg.Decrypted, 0, (int)pg.Size);
#endif
}
}))
{
long len = partitionWrite(decrypted, crc, target, ps, ctx, pc, nkitPartInfo, scrub, hashes, fstFileAlignment);
ps.NewPartitionDataLength = len;
dstPos += len;
lastPart = ps;
}
}
NkitFormat.LogNkitInfo(nkitPartInfo, _log, ps.Id, false);
lastPartitionId = ps.Id;
lastPartitionType = ps.Header.Type;
}
else if (s is WiiFillerSection)
{
WiiFillerSection fs = (WiiFillerSection)s;
ScrubManager scrub = new ScrubManager(null);
JunkStream junk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
if (lastPartitionType == PartitionType.Update && updateRemoved)
{
//preserve the original partition table and update filename
target.Write(removedUpdateFiller.Data, 0, (int)removedUpdateFiller.Size); //remove update partition by adding a 32k placeholder. Avoid having a non update partition at 0x50000
nkitDiscInfo.BytesPreservationDiscPadding += removedUpdateFiller.Size;
nkitDiscInfo.BytesGaps += fs.Size;
dstPos += removedUpdateFiller.Size;
crc.Snapshot(string.Format("{0}Replacement Filler", string.IsNullOrEmpty(lastPartitionId) ? "" : (" " + SourceFiles.CleanseFileName(lastPartitionId).PadRight(4))));
if (extractingUpdate)
{
int storeType;
if ((storeType = disc.WriteRecoveryPartitionFiller(updateCrcStream, junk, fs.DiscOffset, true, true, fs, updateTarget, updateTmpFileName, ref updateFileName, updateCrc, true)) != 0)
{
_log.LogDetail(string.Format("{0}Update recovery partition stored: {1}", storeType == 2 ? "Other " : "", updateFileName));
}
extractingUpdate = false;
}
}
else
{
if (fs.Size != 0)
{
using (StreamCircularBuffer filler = new StreamCircularBuffer(fs.Size, null, null, output =>
{
foreach (WiiFillerSectionItem item in ((WiiFillerSection)s).Sections)
{
output.Write(item.Data, 0, (int)item.Size);
}
}))
{
Gap gap = new Gap(fs.Size, false);
long srcPos = fs.DiscOffset;
long gapLen = gap.Encode(filler, ref srcPos, lastPartitionType == PartitionType.Update ? fs.Size : 0x1cL, fs.Size, junk, scrub, target, _log);
nkitDiscInfo.BytesPreservationData += gapLen;
nkitDiscInfo.BytesGaps += fs.Size;
dstPos += gapLen;
if (lastPart != null)
{
lastPart.NewPartitionDataLength += gapLen;
}
}
}
//pad partition to 32k
int gapLen2 = (int)(dstPos % 0x8000 == 0 ? 0 : 0x8000 - (dstPos % 0x8000));
ByteStream.Zeros.Copy(target, gapLen2);
nkitDiscInfo.BytesPreservationDiscPadding += gapLen2;
dstPos += gapLen2;
if (lastPart != null)
{
lastPart.NewPartitionDataLength += gapLen2;
}
if (lastPart != null)
{
lastPart.Header.WriteUInt32B(0x2bc, (uint)(lastPart.NewPartitionDataLength / 4)); //updates the array in the crc data
hdr.Partitions.First(a => a.DiscOffset == lastPart.DiscOffset).DiscOffset = (lastPart.NewDiscOffset - 0x20000);
}
crc.Snapshot(string.Format("{0}{1}Files+Filler", lastPartitionId ?? "", string.IsNullOrEmpty(lastPartitionId) ? "" : " "));
}
}
}
NkitFormat.LogNkitInfo(nkitDiscInfo, _log, hdr.Id, true);
foreach (CrcItem ci in crc.Crcs.Where(a => a.PatchData != null))
{
ci.PatchCrc = Crc.Compute(ci.PatchData);
}
NCrc readerCrcs;
uint validationCrc;
pc.WriterCheckPoint2Complete(out readerCrcs, out validationCrc, hdr.Data, dstPos); //wait until reader has completed and get crc patches.
if (updateRemoved && crc.Crcs.Length > 2) //freeloader wii only has update partition
{
hdr.RemoveUpdatePartition(crc.Crcs[2].Offset);
}
hdr.UpdateOffsets();
hdr.WriteString(0x200, 8, "NKIT v01"); //header and version
hdr.Write8(0x60, 1);
hdr.Write8(0x61, 1);
hdr.WriteUInt32B(0x208, readerCrcs.FullCrc(true)); //original crc
hdr.WriteUInt32B(0x210, (uint)(imageSize / 4L)); //ctx.ImageLength
hdr.WriteUInt32B(0x218, updateCrc.Value); //Update crc - Only if removed
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data);
crc.Crcs[0].PatchData = hdr.Data;
hdr.WriteUInt32B(0x20C, CrcForce.Calculate(crc.FullCrc(true), dstPos, readerCrcs.FullCrc(true), 0x20C, 0)); //magic to force crc
crc.Crcs[0].PatchCrc = Crc.Compute(hdr.Data); //update with magic applied
pc.WriterCheckPoint3ApplyPatches(crc, false, crc.FullCrc(true), crc.FullCrc(true), this.VerifyIsWrite, "NKit Written");
}
catch (Exception ex)
{
throw pc.SetWriterException(ex, "NkitWriterWii.Write - Convert");
}
}
public virtual OutputResults Process(Context ctx, NStream input, Stream output)
{
OutputResults results = new OutputResults()
{
Conversion = ctx.ConversionName, VerifyOutputResult = VerifyResult.Unverified, ValidateReadResult = VerifyResult.Unverified
};
try
{
StreamCircularBuffer fs = null;
_timer = new Timer
{
Interval = 250
};
_timer.Elapsed += (s, e) =>
{
if (_timerRunning)
{
return; //keep processing
}
try
{
_timerRunning = true;
_log.ProcessingProgress(((IProgress)fs)?.Value ?? 0);
}
catch { }
finally
{
_timerRunning = false;
}
};
_timer.Enabled = true;
long size;
switch (_sizeMode)
{
case ProcessorSizeMode.Source:
size = input.SourceSize;
break;
case ProcessorSizeMode.Stream:
size = input.Length;
break;
case ProcessorSizeMode.Image:
size = input.ImageSize;
break;
case ProcessorSizeMode.Recover:
default:
size = input.RecoverySize;
break;
}
Coordinator pc = new Coordinator(ctx.ValidationCrc, Reader, Writer, size);
pc.Started += (s, e) =>
{
_timer.Enabled = true;
results.AliasJunkId = e.AliasJunkId;
};
pc.Completed += (s, e) =>
{
_timer.Enabled = false;
if (Writer is HashWriter)
{
results.OutputMd5 = e.Md5;
results.OutputSha1 = e.Sha1;
}
else
{
MemorySection hdr = new MemorySection(e.Header ?? input.DiscHeader.Data);
results.OutputTitle = hdr.ReadStringToNull(0x20, 0x60);
results.OutputDiscNo = hdr.Read8(6);
results.OutputDiscVersion = hdr.Read8(7);
results.OutputId8 = string.Concat(hdr.ReadString(0, 6), results.OutputDiscNo.ToString("X2"), results.OutputDiscVersion.ToString("X2"));
results.ProcessorMessage = e.ResultMessage;
results.OutputCrc = e.PatchedCrc;
results.IsRecoverable = e.IsRecoverable;
if (e.ValidationCrc != 0)
{
results.ValidationCrc = e.ValidationCrc;
results.ValidateReadResult = e.ValidationCrc == e.PatchedCrc ? VerifyResult.VerifySuccess : VerifyResult.VerifyFailed;
}
if (!(Writer is VerifyWriter))
{
results.OutputSize = e.OutputSize; //never store the verify size
}
else
{
results.VerifyCrc = e.VerifyCrc;
if (e.VerifyIsWrite) //e.ValidationCrc can be set from a previous process run
{
results.VerifyOutputResult = results.ValidationCrc == results.VerifyCrc ? VerifyResult.VerifySuccess : VerifyResult.VerifyFailed;
}
}
}
bool l9 = pc.Patches.Crcs.Any(a => a.Offset > 0xFFFFFFFFL || a.Length > 0xFFFFFFFFL);
if (pc.ReaderCrcs != null)
{
foreach (CrcItem c in pc.ReaderCrcs.Crcs)
{
_log.LogDebug(string.Format("R-CRC {0} Before:{1} After:{2} L:{3} {4}", c.Offset.ToString(l9 ? "X9" : "X8"), c.Value.ToString("X8"), c.PatchCrc == 0 ? " " : c.PatchCrc.ToString("X8"), c.Length.ToString(l9 ? "X9" : "X8"), SourceFiles.CleanseFileName(c.Name)));
}
_log.LogDebug(string.Format("ReadCRC {0}Before:{1} After:{2}", l9 ? " " : "", pc.ReaderCrcs.FullCrc(false).ToString("X8"), pc.ReaderCrcs.FullCrc(true).ToString("X8")));
}
if (pc.WriterCrcs != null)
{
foreach (CrcItem c in pc.WriterCrcs.Crcs)
{
_log.LogDebug(string.Format("W-CRC {0} Before:{1} After:{2} L:{3} {4}", c.Offset.ToString(l9 ? "X9" : "X8"), c.Value.ToString("X8"), c.PatchCrc == 0 ? " " : c.PatchCrc.ToString("X8"), c.Length.ToString(l9 ? "X9" : "X8"), SourceFiles.CleanseFileName(c.Name)));
}
_log.LogDebug(string.Format("WriteCRC {0}Before:{1} After:{2}", l9 ? " " : "", pc.WriterCrcs.FullCrc(false).ToString("X8"), pc.WriterCrcs.FullCrc(true).ToString("X8")));
}
_log.ProcessingComplete(results.OutputSize, results.ProcessorMessage, true);
};
try
{
_log.ProcessingStart(input.SourceSize, Title);
using (fs = new StreamCircularBuffer(size, input, null, s => Reader.Read(ctx, input, s, pc))) //read in stream and write to circular buffer
{
Writer.Write(ctx, fs, output, pc);
}
}
catch
{
if (pc.Exception != null)
{
throw pc.Exception;
}
if (fs.WriterException != null)
{
throw fs.WriterException;
}
throw; //writer exception
}
foreach (CrcItem crc in pc.Patches.Crcs.Where(a => a.PatchData != null || a.PatchFile != null))
{
output.Seek(crc.Offset, SeekOrigin.Begin);
if (crc.PatchFile == null)
{
output.Write(crc.PatchData, 0, (int)Math.Min(crc.Length, crc.PatchData.Length)); //PatchData might be larger
}
else
{
using (FileStream pf = File.OpenRead(crc.PatchFile))
{
pf.Copy(output, pf.Length);
ByteStream.Zeros.Copy(output, crc.Length - pf.Length);
}
}
}
}
catch (Exception ex)
{
if (_timer != null)
{
_timer.Enabled = false;
}
try
{
_log.ProcessingComplete(results.OutputSize, results.ProcessorMessage, false); // force any log lines to be output - handy for diagnosis
}
catch { }
throw new HandledException(ex, "Failed processing {0} -> {1}", Reader?.GetType()?.Name ?? "<null>", Writer?.GetType()?.Name ?? "<null>");
}
finally
{
if (_timer != null)
{
_timer.Enabled = false;
_timer = null;
}
}
return(results);
}
private OutputResults process(string conversion, SourceFile sourceFile, bool renameWithMasks, bool toNkit, NkitFormatType nkitFormat, bool isRecovery, bool fullVerify, bool calcHashes, bool testMode, Func <NStream, IEnumerable <Processor> > passes)
{
OutputResults results = null;
NStream nstream = _nstream;
string lastTmp = null;
string tmp = null;
this.ConvertionName = conversion;
try
{
SourceFile sf = null;
long sourceSize = nstream.SourceSize;
_context = new Context();
_context.Initialise(this.ConvertionName, sourceFile, _settings, true, isRecovery, nstream.IsGameCube, nstream.Id8, this);
List <Processor> processors = passes(nstream).Where(a => a != null).ToList();
if (nkitFormat == NkitFormatType.Gcz)
{
processors.Add(new Processor(new IsoReader(), new GczWriter(), "To GCZ", this, false, true, ProcessorSizeMode.Stream));
}
if (calcHashes)
{
processors.Add(new Processor(new IsoReader(), new HashWriter(), "Calc Hashes", this, false, true, ProcessorSizeMode.Stream));
}
if (fullVerify)
{
if (!toNkit)
{
processors.Add(new Processor(new IsoReader(), new VerifyWriter(), "Full Verify", this, false, true, ProcessorSizeMode.Stream));
}
else
{
processors.Add(new Processor(nstream.IsGameCube ? new NkitReaderGc() : (IReader) new NkitReaderWii(), new VerifyWriter(), "NKit Verify", this, true, true, ProcessorSizeMode.Image));
}
processors.Last().Writer.RequireVerifyCrc = true;
processors.Last().Writer.VerifyIsWrite = true; //read verify
}
_totalPasses = processors.Count();
if (processors.Count == 0)
{
return(null);
}
DateTime dt = DateTime.Now;
_completedPasses = 0;
Log("PROCESSING" + (testMode ? " [TEST MODE]" : ((_context.Settings.DeleteSource ? " [DELETE SOURCE]" : ""))));
Log("-------------------------------------------------------------------------------");
if (_forcedWiiFullNkitRebuild)
{
LogBlank();
Log(string.Format("Nkit Reencode forced: NkitUpdatePartitionRemoval is {0} and source image has {1} Update Partition", _context.Settings.NkitUpdatePartitionRemoval.ToString(), nstream.IsNkitUpdateRemoved ? "no" : "an"));
LogBlank();
}
Log(string.Format("{0} [{1}] {2} [MiB:{3,2:#.0}]", friendly(nstream.Title), friendly(nstream.Id), nstream.IsGameCube ? "GameCube" : "Wii", (sourceSize / (double)(1024 * 1024))));
LogBlank();
string passesText = getPassesLine(nstream, processors);
Log(passesText);
int i = 1;
foreach (Processor pro in processors.Where(pro => pro != null))
{
LogDebug(string.Format("Pass {0}: {1}", (i++).ToString(), pro.ToString()));
}
LogBlank();
foreach (Processor pro in processors)
{
//sort out temp file and open input as nstream each time
//raise progress and output messages from processors
if (sf != null)
{
nstream = sf.OpenNStream(!(pro.Writer is HashWriter)); //if hashWriter then read as raw file
sf = null;
}
tmp = null;
FileStream tmpFs = null;
try
{
if (pro.HasWriteStream)
{
tmp = Path.Combine(_context.Settings.TempPath, Path.GetFileName(Path.GetTempFileName()));
if (!Directory.Exists(_context.Settings.TempPath))
{
Directory.CreateDirectory(_context.Settings.TempPath);
}
tmpFs = File.Create(tmp, 0x400 * 0x400 * 4, FileOptions.SequentialScan);
}
_logCache = new List <Tuple <string, LogMessageType> >();
OutputResults nr = pro.Process(_context, nstream, tmpFs ?? Stream.Null);
_logCache = null;
if (results == null)
{
results = nr;
results.DiscType = nstream.IsGameCube ? DiscType.GameCube : DiscType.Wii;
results.InputFileName = sourceFile.AllFiles.Length != 0 ? sourceFile.AllFiles[0] : sourceFile.FilePath;
results.InputDiscNo = nstream.DiscNo;
results.InputDiscVersion = nstream.Version;
results.InputTitle = nstream.Title;
results.InputId8 = nstream.Id8;
results.InputSize = sourceSize;
results.FullSize = nstream.ImageSize;
results.Passes = passesText;
if (pro.IsVerify)
{
results.OutputSize = results.InputSize;
}
}
else if (pro.Writer is HashWriter) //hash writer gives no meaningful info back other than md5 and sha1 (the crc is forced when nkit, so ignore)
{
results.OutputMd5 = nr.OutputMd5;
results.OutputSha1 = nr.OutputSha1;
}
else
{
if (nr.AliasJunkId != null)
{
results.AliasJunkId = nr.AliasJunkId;
}
if (nr.OutputTitle != null)
{
results.OutputDiscNo = nr.OutputDiscNo;
results.OutputDiscVersion = nr.OutputDiscVersion;
results.OutputTitle = nr.OutputTitle;
}
results.OutputId8 = nr.OutputId8;
results.OutputCrc = nr.OutputCrc;
results.OutputPrePatchCrc = nr.OutputPrePatchCrc;
results.FullSize = nstream.ImageSize;
if (tmp != null)
{
results.OutputSize = nr.OutputSize;
}
if (nr.ValidationCrc != 0 && results.VerifyCrc != 0)
{
results.VerifyCrc = 0; //blank verify if a new ValidationCrc is set - verification happened when both != 0
}
if (nr.VerifyCrc != 0)
{
results.VerifyCrc = nr.VerifyCrc;
}
if (nr.ValidationCrc != 0)
{
results.ValidationCrc = nr.ValidationCrc;
}
if (nr.ValidateReadResult != VerifyResult.Unverified)
{
results.ValidateReadResult = nr.ValidateReadResult;
}
if (nr.VerifyOutputResult != VerifyResult.Unverified)
{
if (results.ValidateReadResult == VerifyResult.Unverified && nstream.IsNkit)
{
results.ValidateReadResult = nr.VerifyOutputResult;
}
results.VerifyOutputResult = nr.VerifyOutputResult;
}
if (nr.IsRecoverable)
{
results.IsRecoverable = nr.IsRecoverable;
}
}
}
finally
{
if (tmpFs != null)
{
tmpFs.Dispose();
}
nstream.Close();
}
if (lastTmp != null && tmp != null)
{
File.Delete(lastTmp);
}
//handle failures well
if (results.ValidateReadResult == VerifyResult.VerifyFailed || results.VerifyOutputResult == VerifyResult.VerifyFailed)
{
lastTmp = tmp; //keep post loop happy
break;
}
if (_completedPasses != _totalPasses) //last item
{
sf = SourceFiles.OpenFile(tmp ?? lastTmp);
}
if (tmp != null)
{
lastTmp = tmp;
}
}
TimeSpan ts = DateTime.Now - dt;
results.ProcessingTime = ts;
//FAIL
if (results.ValidateReadResult == VerifyResult.VerifyFailed || results.VerifyOutputResult == VerifyResult.VerifyFailed)
{
LogBlank();
Log(string.Format("Verification Failed Crc:{0} - Failed Test Crc:{1}", results.OutputCrc.ToString("X8"), results.ValidationCrc.ToString("X8")));
if (lastTmp != null) //only null when verify only
{
Log("Deleting Output" + (Settings.OutputLevel != 3 ? "" : " (Skipped as OutputLevel is 3:Debug)"));
results.OutputFileName = null;
if (Settings.OutputLevel != 3)
{
File.Delete(lastTmp);
}
LogBlank();
}
}
else //SUCCESS
{
LogBlank();
Log(string.Format("Completed ~ {0}m {1}s [MiB:{2:#.0}]", ((int)ts.TotalMinutes).ToString(), ts.Seconds.ToString(), (results.OutputSize / (double)(1024 * 1024))));
LogBlank();
Log("RESULTS");
Log("-------------------------------------------------------------------------------");
uint finalCrc = results.ValidationCrc != 0 ? results.ValidationCrc : results.OutputCrc;
string mask = _context.Settings.MatchFailRenameToMask;
results.OutputFileExt = "." + SourceFiles.ExtensionString(false, false, toNkit, nkitFormat == NkitFormatType.Gcz).ToLower();
results.RedumpInfo = _context.Dats.GetRedumpEntry(_context.Settings, _context.Dats, finalCrc);
if (results.RedumpInfo.MatchType == MatchType.Redump || results.RedumpInfo.MatchType == MatchType.Custom)
{
Log(string.Format("{0} [{1} Name]", results.RedumpInfo.MatchName, results.RedumpInfo.MatchType.ToString()));
if (results.IsRecoverable)
{
Log(string.Format("Missing Recovery data is required to correctly restore this image!", results.RedumpInfo.MatchName, results.RedumpInfo.MatchType.ToString()));
}
mask = results.RedumpInfo.MatchType == MatchType.Custom ? _context.Settings.CustomMatchRenameToMask : _context.Settings.RedumpMatchRenameToMask;
}
else
{
Log(string.Format("CRC {0} not in Redump or Custom Dat", finalCrc.ToString("X8")));
}
LogBlank();
outputResults(results);
if (lastTmp != null) //only null when verify only
{
if (testMode)
{
Log("TestMode: Deleting Output");
results.OutputFileName = null;
if (File.Exists(lastTmp))
{
File.Delete(lastTmp);
}
}
else if (isRecovery && _context.Settings.RecoveryMatchFailDelete && results.RedumpInfo.MatchType == MatchType.MatchFail)
{
Log("Failed to Recover to Dat Entry: Deleting Output");
results.OutputFileName = null;
File.Delete(lastTmp);
}
else
{
if (renameWithMasks)
{
results.OutputFileName = _context.Dats.GetFilename(results, mask);
Log("Renaming Output Using Masks");
}
else
{
results.OutputFileName = SourceFiles.GetUniqueName(sourceFile.CreateOutputFilename(results.OutputFileExt));
Log("Renaming Output Based on Source File" + (sourceFile.AllFiles.Count() > 1 ? "s" : ""));
}
LogBlank();
string path = Path.GetDirectoryName(results.OutputFileName);
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
File.Move(lastTmp, results.OutputFileName);
Log(string.Format("Output: {0}", Path.GetDirectoryName(results.OutputFileName)));
Log(string.Format(" {0}", Path.GetFileName(results.OutputFileName)));
//double check test mode just to be sure
if (_context.Settings.DeleteSource && !testMode && results.VerifyOutputResult == VerifyResult.VerifySuccess)
{
LogBlank();
Log("Deleting Source:");
foreach (string s in sourceFile.AllFiles.Length == 0 ? new string[] { sourceFile.FilePath } : sourceFile.AllFiles)
{
Log(string.Format(" {0}", s));
File.Delete(s);
}
}
}
LogBlank();
}
}
}
catch (Exception ex)
{
try
{
if (lastTmp == null)
{
lastTmp = tmp;
}
if (lastTmp != null)
{
LogBlank();
Log("Deleting Output" + (Settings.OutputLevel != 3 ? "" : " (Skipped as OutputLevel is 3:Debug)"));
if (results != null)
{
results.OutputFileName = null;
}
if (Settings.OutputLevel != 3)
{
File.Delete(lastTmp);
}
}
}
catch { }
if (_context.Settings.EnableSummaryLog)
{
if (results == null)
{
results = new OutputResults();
results.Conversion = this.ConvertionName;
results.DiscType = (nstream?.IsGameCube ?? true) ? DiscType.GameCube : DiscType.Wii;
results.InputFileName = (sourceFile?.AllFiles?.Length ?? 0) == 0 ? (sourceFile?.FilePath ?? "") : (sourceFile?.AllFiles.FirstOrDefault() ?? "");
results.InputDiscNo = nstream?.DiscNo ?? 0;
results.InputDiscVersion = nstream?.Version ?? 0;
results.InputTitle = nstream?.Title ?? "";
results.InputId8 = nstream?.Id8 ?? "";
results.InputSize = sourceFile?.Length ?? 0;
}
results.VerifyOutputResult = VerifyResult.Error;
HandledException hex = ex as HandledException;
if (hex == null)
{
hex = new HandledException(ex, "Unhandled Exception");
}
results.ErrorMessage = hex.FriendlyErrorMessage;
}
throw;
}
finally
{
if (_context.Settings.EnableSummaryLog)
{
summaryLog(_context.Settings, results);
Log("Summary Log Written" + (results.VerifyOutputResult != VerifyResult.Error ? "" : " as Errored!"));
LogBlank();
}
}
return(results);
}
internal long WriteRecoveryPartitionData(Stream crcStream, bool unscrub, WiiPartitionSection ps, int channelNo, out string tempFileName, out string fileName, out NStream output)
{
if (ps.Header.Type == PartitionType.Update)
{
fileName = string.Format("{0}_{1}_", BitConverter.ToString(ps.Header.ContentSha1).Replace("-", ""), ps.Header.IsKorean ? "K" : "N");
}
else
{
fileName = string.Format("{0}_{1}_{2}_{3}_", this.NStream.Id8, channelNo.ToString().PadLeft(2, '0'), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4), ps.Header.IsKorean ? "K" : "N");
}
tempFileName = Path.Combine(Settings.OtherRecoveryFilesPath, fileName + "TEMP");
Directory.CreateDirectory(Settings.OtherRecoveryFilesPath);
output = new NStream(File.Create(tempFileName));
output.Write(ps.Header.Data, 0, (int)ps.Header.Size);
crcStream.Write(ps.Header.Data, 0, (int)ps.Header.Size);
long read = ps.Header.Size;
foreach (WiiPartitionGroupSection pg in ps.Sections)
{
if (unscrub)
{
pg.Unscrub(null);
}
output.Write(pg.Encrypted, 0, (int)pg.Size);
crcStream.Write(pg.Encrypted, 0, (int)pg.Size);
read = ps.Size;
}
return(read);
}
/// <summary>
/// Extracts files from partitions, this is not random access. The iso is read in it's entirety
/// </summary>
public ExtractResult ExtractRecoveryFilesWii()
{
List <ExtractRecoveryResult> result = new List <ExtractRecoveryResult>();
List <WiiPartitionInfo> toExtract = new List <WiiPartitionInfo>();
WiiDiscHeaderSection hdr = null;
WiiPartitionHeaderSection pHdr = null;
NStream target = null;
bool extracting = false;
int channel = 1;
string lastPartitionId = null;
PartitionType lastPartitionType = PartitionType.Other;
string fileName = null;
string tmpFileName = null;
int extracted = 0;
Crc crc = new Crc();
Stream crcStream = null;
long imageSize = this.NStream.RecoverySize; //for Wii: pHdr.PartitionDataLength
bool isIso = false; //force to always scrub //Path.GetExtension(_name).ToLower() == ".iso";
foreach (IWiiDiscSection s in this.EnumerateSectionsFix(false, true, false))
{
if (s is WiiDiscHeaderSection)
{
hdr = (WiiDiscHeaderSection)s;
Log?.Log(string.Format("Processing: {0}", SourceFileName));
toExtract = hdr.Partitions.Where(a => a.Type != PartitionType.Data && a.Type != PartitionType.GameData).ToList();
if (toExtract.Count() == 0)
{
Log?.Log(string.Format(" Skipped: No Recovery Partitions to extract - {0}", SourceFileName));
break;
}
}
else if (s is WiiPartitionSection)
{
WiiPartitionSection ps = (WiiPartitionSection)s;
if (!toExtract.Any(a => a.DiscOffset == ps.DiscOffset))
{
Log?.Log(string.Format(" Skipping {0} Partition: [{1}]...", ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));
continue;
}
extracting = true;
pHdr = ps.Header;
extracting = true;
Log?.Log(string.Format(" {0} of {1} - Extracting {2} Recovery Partition: [{3}]", (extracted + 1).ToString(), toExtract.Count().ToString(), ps.Header.Type.ToString(), SourceFiles.CleanseFileName(ps.Header.Id).PadRight(4)));
crcStream = new CryptoStream(ByteStream.Zeros, crc, CryptoStreamMode.Write);
crc.Initialize();
WriteRecoveryPartitionData(crcStream, isIso, ps, channel, out tmpFileName, out fileName, out target);
lastPartitionId = ps.Id;
lastPartitionType = ps.Header.Type;
}
else if (s is WiiFillerSection)
{
JunkStream junk = new JunkStream(lastPartitionType != PartitionType.Data ? hdr.ReadString(0, 4) : lastPartitionId, hdr.Read8(6), lastPartitionType == PartitionType.Update ? 0 : imageSize);
WiiFillerSection fs = (WiiFillerSection)s;
if (extracting)
{
int storeType;
if ((storeType = WriteRecoveryPartitionFiller(crcStream, junk, fs.DiscOffset, pHdr.Type == PartitionType.Update, false, fs, target, tmpFileName, ref fileName, crc, false)) != 0)
{
result.Add(new ExtractRecoveryResult()
{
FileName = fileName, Extracted = true, Type = PartitionType.Other, IsNew = storeType == 2, IsGameCube = false
});
}
if (pHdr.Type != PartitionType.Update)
{
channel++;
}
extracted++;
extracting = false;
bool complete = (extracted == toExtract.Count());
if (complete)
{
break;
}
}
}
}
return(createExtractResult((Region)hdr.ReadUInt32B(0x4e000), result.ToArray()));
}
private static string pathFix(string path)
{
return(SourceFiles.PathFix(path));
}
public string ExtensionString()
{
return(SourceFiles.ExtensionString(_isIsoDec, _isWbfs, _isNkit, _isGcz));
}