private void outputResults(OutputResults r) { bool output = false; if (r.InputId4 != r.OutputId4) { Log(string.Format("ID changed from {0} to {1}", r.InputId4, r.OutputId4)); output = true; } if (r.InputTitle != r.OutputTitle) { Log(string.Format("Title changed from '{0}' to '{1}'", r.InputTitle, r.OutputTitle)); output = true; } if (r.InputDiscVersion != r.OutputDiscVersion) { Log(string.Format("Version changed from v1.{0} to v1.{1}", r.InputDiscVersion.ToString("D2"), r.OutputDiscVersion.ToString("D2"))); output = true; } if (r.InputDiscNo != r.OutputDiscNo) { Log(string.Format("Disc No. changed from {0} to {1}", r.InputDiscNo.ToString(), r.OutputDiscNo.ToString())); output = true; } if (output) { LogBlank(); } }
private void btnSettingsProcess_Click(object sender, EventArgs e) { ListViewItem[] items = itemsToProcess().ToArray(); if (_state == State.New) //if not resuming { _processFileIndex = 0; _processFilesCount = items.Length; } setPaths(txtSettingsOutputPathBase.Text); _state = State.Processing; setScreenState(); process(cboSettingsMode.SelectedIndex, items, item => //started item { Invoke((MethodInvoker) delegate { if (lvw.SelectedItems.Count == 0 || lvw.SelectedItems[0] == _processingItem) { item.Selected = true; //move the logging on } _processingItem = item; }); }, item => //completed item { Invoke((MethodInvoker) delegate { setLog(((ProcessFile)item.Tag).Log); _processFileIndex++; OutputResults results = ((ProcessFile)item.Tag).Results; if (results != null) { item.SubItems[1].Text = results.DiscType.ToString(); item.SubItems[2].Text = results.ValidateReadResult.ToString(); item.SubItems[3].Text = results.VerifyOutputResult.ToString(); item.SubItems[4].Text = results.OutputCrc.ToString("X8") ?? ""; item.SubItems[5].Text = SourceFiles.CleanseFileName(results.OutputId4 ?? ""); item.SubItems[6].Text = (results.RedumpInfo?.MatchType.ToString() ?? "") + (results.IsRecoverable ? "Recoverable" : ""); item.SubItems[7].Text = (results.OutputSize / (double)(1024 * 1024)).ToString("#.0") + " MiB"; item.SubItems[8].Text = results.RedumpInfo?.MatchName ?? ""; } else { item.SubItems[1].Text = "Error"; } }); return(_state != State.Stopping); //stop }).ContinueWith( t => //finished processing { _state = _processFileIndex == _processFilesCount ? State.Complete : State.Stopped; setScreenState(); }); }
public string GetFilename(OutputResults results, string mask) { try { string fileName = SourceFiles.CleanseFileName(SourceFiles.RemoveExtension(results.InputFileName, true)); string titleName = SourceFiles.CleanseFileName(results.OutputTitle) ?? fileName; string tgdbName = SourceFiles.CleanseFileName(GameTdbData.FirstOrDefault(a => a.Item1 == results.OutputId6)?.Item2) ?? titleName; string matchName = results.RedumpInfo?.MatchName != null?SourceFiles.CleanseFileName(results.RedumpInfo.MatchName) : (tgdbName ?? fileName); Dictionary <string, string> values = new Dictionary <string, string> { { "%src", results.InputFileName ?? Path.GetDirectoryName(Assembly.GetEntryAssembly().Location) }, { "%nmo", fileName }, { "%nmg", tgdbName }, { "%nmd", titleName }, { "%nmm", matchName }, { "%dno", results.OutputDiscNo == 0 ? "" : string.Format("(Disc {0})", results.OutputDiscNo.ToString()) }, { "%ver", results.OutputDiscVersion == 0 ? "" : string.Format("(v1.{0})", results.OutputDiscVersion.ToString("D2")) }, { "%rev", results.OutputDiscVersion == 0 ? "" : string.Format("(Rev {0})", results.OutputDiscVersion.ToString("D2")) }, { "%crc", results.OutputCrc.ToString("X8") }, { "%md5", results.OutputMd5 == null ? "" : BitConverter.ToString(results.OutputMd5).Replace("-", "") }, { "%sha", results.OutputSha1 == null ? "" : BitConverter.ToString(results.OutputSha1).Replace("-", "") }, { "%id4", SourceFiles.CleanseFileName(results.OutputId4).PadRight(4) }, { "%id6", SourceFiles.CleanseFileName(results.OutputId6).PadRight(6) }, { "%id8", SourceFiles.CleanseFileName(results.OutputId8).PadRight(8) } }; string[] braces = new[] { "%crc", "%md5", "%sha", "%id4", "%id6", "%id8" }; string fn = mask; foreach (KeyValuePair <string, string> v in values) { if (fn.Contains(v.Key)) { if (string.IsNullOrEmpty(v.Value) || Regex.IsMatch(fn, Regex.Escape(v.Value), RegexOptions.IgnoreCase)) { fn = Regex.Replace(fn, string.Format(@"[ ._-]?{0}", v.Key), "", RegexOptions.IgnoreCase); } else { fn = fn.Replace(v.Key, string.Concat(braces.Contains(v.Key) ? "[" : "", v.Value, braces.Contains(v.Key) ? "]" : "")); } } } return(SourceFiles.GetUniqueName(fn.Replace("%ext", results.OutputFileExt.Trim('.')))); } catch (Exception ex) { throw new HandledException(ex, "DatData.GetFilename - Replace masks"); } }
//////////////////////////////////////////////////////////////// // OUTPUT STUFF //////////////////////////////////////////////////////////////// private void summaryLog(Settings settings, OutputResults results) { try { if (!File.Exists(settings.SummaryLog)) { System.IO.File.AppendAllText(settings.SummaryLog, string.Join("\t", "TimeStamp", "Conversion", "System", "ReadResult", "OutputResult", "OutputCrc", "OutputID4", "RedumpMatch", "RedumpName", "InputSize", "OutputSize", "FullSize", "InputFilename", "OutputFilename", "MD5", "SHA1", "Passes", "SecondsElapsed", "ErrorMessage") + Environment.NewLine); } if (settings.EnableSummaryLog) { System.IO.File.AppendAllText(settings.SummaryLog, string.Join("\t", DateTime.Now.ToString(), results.Conversion, results.DiscType.ToString(), results.ValidateReadResult.ToString(), results.VerifyOutputResult.ToString(), results.OutputCrc.ToString("X8") ?? "", results.OutputId4 ?? "", (results.RedumpInfo?.MatchType.ToString() ?? "") + (results.IsRecoverable ? "Recoverable" : ""), results.RedumpInfo?.MatchName ?? "", results.InputSize.ToString(), results.OutputSize.ToString(), results.FullSize.ToString(), results.InputFileName ?? "", results.OutputFileName ?? "", results.OutputMd5 == null ? "" : BitConverter.ToString(results.OutputMd5).Replace("-", ""), results.OutputSha1 == null ? "" : BitConverter.ToString(results.OutputSha1).Replace("-", ""), results.Passes ?? "", ((int)results.ProcessingTime.TotalSeconds).ToString(), (results.ErrorMessage ?? "").Replace("\r", "").Replace('\t', ' ').Trim('\n', ' ').Replace("\n", " : ") ) + Environment.NewLine); } } catch { } }
public virtual OutputResults Process(Context ctx, NStream input, Stream output) { OutputResults results = new OutputResults() { Conversion = ctx.ConversionName, VerifyOutputResult = VerifyResult.Unverified, ValidateReadResult = VerifyResult.Unverified }; try { StreamCircularBuffer fs = null; _timer = new Timer { Interval = 250 }; _timer.Elapsed += (s, e) => { if (_timerRunning) { return; //keep processing } try { _timerRunning = true; _log.ProcessingProgress(((IProgress)fs)?.Value ?? 0); } catch { } finally { _timerRunning = false; } }; _timer.Enabled = true; long size; switch (_sizeMode) { case ProcessorSizeMode.Source: size = input.SourceSize; break; case ProcessorSizeMode.Stream: size = input.Length; break; case ProcessorSizeMode.Image: size = input.ImageSize; break; case ProcessorSizeMode.Recover: default: size = input.RecoverySize; break; } Coordinator pc = new Coordinator(ctx.ValidationCrc, Reader, Writer, size); pc.Started += (s, e) => { _timer.Enabled = true; results.AliasJunkId = e.AliasJunkId; }; pc.Completed += (s, e) => { _timer.Enabled = false; if (Writer is HashWriter) { results.OutputMd5 = e.Md5; results.OutputSha1 = e.Sha1; } else { MemorySection hdr = new MemorySection(e.Header ?? input.DiscHeader.Data); results.OutputTitle = hdr.ReadStringToNull(0x20, 0x60); results.OutputDiscNo = hdr.Read8(6); results.OutputDiscVersion = hdr.Read8(7); results.OutputId8 = string.Concat(hdr.ReadString(0, 6), results.OutputDiscNo.ToString("X2"), results.OutputDiscVersion.ToString("X2")); results.ProcessorMessage = e.ResultMessage; results.OutputCrc = e.PatchedCrc; results.IsRecoverable = e.IsRecoverable; if (e.ValidationCrc != 0) { results.ValidationCrc = e.ValidationCrc; results.ValidateReadResult = e.ValidationCrc == e.PatchedCrc ? VerifyResult.VerifySuccess : VerifyResult.VerifyFailed; } if (!(Writer is VerifyWriter)) { results.OutputSize = e.OutputSize; //never store the verify size } else { results.VerifyCrc = e.VerifyCrc; if (e.VerifyIsWrite) //e.ValidationCrc can be set from a previous process run { results.VerifyOutputResult = results.ValidationCrc == results.VerifyCrc ? VerifyResult.VerifySuccess : VerifyResult.VerifyFailed; } } } bool l9 = pc.Patches.Crcs.Any(a => a.Offset > 0xFFFFFFFFL || a.Length > 0xFFFFFFFFL); if (pc.ReaderCrcs != null) { foreach (CrcItem c in pc.ReaderCrcs.Crcs) { _log.LogDebug(string.Format("R-CRC {0} Before:{1} After:{2} L:{3} {4}", c.Offset.ToString(l9 ? "X9" : "X8"), c.Value.ToString("X8"), c.PatchCrc == 0 ? " " : c.PatchCrc.ToString("X8"), c.Length.ToString(l9 ? "X9" : "X8"), SourceFiles.CleanseFileName(c.Name))); } _log.LogDebug(string.Format("ReadCRC {0}Before:{1} After:{2}", l9 ? " " : "", pc.ReaderCrcs.FullCrc(false).ToString("X8"), pc.ReaderCrcs.FullCrc(true).ToString("X8"))); } if (pc.WriterCrcs != null) { foreach (CrcItem c in pc.WriterCrcs.Crcs) { _log.LogDebug(string.Format("W-CRC {0} Before:{1} After:{2} L:{3} {4}", c.Offset.ToString(l9 ? "X9" : "X8"), c.Value.ToString("X8"), c.PatchCrc == 0 ? " " : c.PatchCrc.ToString("X8"), c.Length.ToString(l9 ? "X9" : "X8"), SourceFiles.CleanseFileName(c.Name))); } _log.LogDebug(string.Format("WriteCRC {0}Before:{1} After:{2}", l9 ? " " : "", pc.WriterCrcs.FullCrc(false).ToString("X8"), pc.WriterCrcs.FullCrc(true).ToString("X8"))); } _log.ProcessingComplete(results.OutputSize, results.ProcessorMessage, true); }; try { _log.ProcessingStart(input.SourceSize, Title); using (fs = new StreamCircularBuffer(size, input, null, s => Reader.Read(ctx, input, s, pc))) //read in stream and write to circular buffer { Writer.Write(ctx, fs, output, pc); } } catch { if (pc.Exception != null) { throw pc.Exception; } if (fs.WriterException != null) { throw fs.WriterException; } throw; //writer exception } foreach (CrcItem crc in pc.Patches.Crcs.Where(a => a.PatchData != null || a.PatchFile != null)) { output.Seek(crc.Offset, SeekOrigin.Begin); if (crc.PatchFile == null) { output.Write(crc.PatchData, 0, (int)Math.Min(crc.Length, crc.PatchData.Length)); //PatchData might be larger } else { using (FileStream pf = File.OpenRead(crc.PatchFile)) { pf.Copy(output, pf.Length); ByteStream.Zeros.Copy(output, crc.Length - pf.Length); } } } } catch (Exception ex) { if (_timer != null) { _timer.Enabled = false; } try { _log.ProcessingComplete(results.OutputSize, results.ProcessorMessage, false); // force any log lines to be output - handy for diagnosis } catch { } throw new HandledException(ex, "Failed processing {0} -> {1}", Reader?.GetType()?.Name ?? "<null>", Writer?.GetType()?.Name ?? "<null>"); } finally { if (_timer != null) { _timer.Enabled = false; _timer = null; } } return(results); }
private OutputResults process(string conversion, SourceFile sourceFile, bool renameWithMasks, bool toNkit, NkitFormatType nkitFormat, bool isRecovery, bool fullVerify, bool calcHashes, bool testMode, Func <NStream, IEnumerable <Processor> > passes) { OutputResults results = null; NStream nstream = _nstream; string lastTmp = null; string tmp = null; this.ConvertionName = conversion; try { SourceFile sf = null; long sourceSize = nstream.SourceSize; _context = new Context(); _context.Initialise(this.ConvertionName, sourceFile, _settings, true, isRecovery, nstream.IsGameCube, nstream.Id8, this); List <Processor> processors = passes(nstream).Where(a => a != null).ToList(); if (nkitFormat == NkitFormatType.Gcz) { processors.Add(new Processor(new IsoReader(), new GczWriter(), "To GCZ", this, false, true, ProcessorSizeMode.Stream)); } if (calcHashes) { processors.Add(new Processor(new IsoReader(), new HashWriter(), "Calc Hashes", this, false, true, ProcessorSizeMode.Stream)); } if (fullVerify) { if (!toNkit) { processors.Add(new Processor(new IsoReader(), new VerifyWriter(), "Full Verify", this, false, true, ProcessorSizeMode.Stream)); } else { processors.Add(new Processor(nstream.IsGameCube ? new NkitReaderGc() : (IReader) new NkitReaderWii(), new VerifyWriter(), "NKit Verify", this, true, true, ProcessorSizeMode.Image)); } processors.Last().Writer.RequireVerifyCrc = true; processors.Last().Writer.VerifyIsWrite = true; //read verify } _totalPasses = processors.Count(); if (processors.Count == 0) { return(null); } DateTime dt = DateTime.Now; _completedPasses = 0; Log("PROCESSING" + (testMode ? " [TEST MODE]" : ((_context.Settings.DeleteSource ? " [DELETE SOURCE]" : "")))); Log("-------------------------------------------------------------------------------"); if (_forcedWiiFullNkitRebuild) { LogBlank(); Log(string.Format("Nkit Reencode forced: NkitUpdatePartitionRemoval is {0} and source image has {1} Update Partition", _context.Settings.NkitUpdatePartitionRemoval.ToString(), nstream.IsNkitUpdateRemoved ? "no" : "an")); LogBlank(); } Log(string.Format("{0} [{1}] {2} [MiB:{3,2:#.0}]", friendly(nstream.Title), friendly(nstream.Id), nstream.IsGameCube ? "GameCube" : "Wii", (sourceSize / (double)(1024 * 1024)))); LogBlank(); string passesText = getPassesLine(nstream, processors); Log(passesText); int i = 1; foreach (Processor pro in processors.Where(pro => pro != null)) { LogDebug(string.Format("Pass {0}: {1}", (i++).ToString(), pro.ToString())); } LogBlank(); foreach (Processor pro in processors) { //sort out temp file and open input as nstream each time //raise progress and output messages from processors if (sf != null) { nstream = sf.OpenNStream(!(pro.Writer is HashWriter)); //if hashWriter then read as raw file sf = null; } tmp = null; FileStream tmpFs = null; try { if (pro.HasWriteStream) { tmp = Path.Combine(_context.Settings.TempPath, Path.GetFileName(Path.GetTempFileName())); if (!Directory.Exists(_context.Settings.TempPath)) { Directory.CreateDirectory(_context.Settings.TempPath); } tmpFs = File.Create(tmp, 0x400 * 0x400 * 4, FileOptions.SequentialScan); } _logCache = new List <Tuple <string, LogMessageType> >(); OutputResults nr = pro.Process(_context, nstream, tmpFs ?? Stream.Null); _logCache = null; if (results == null) { results = nr; results.DiscType = nstream.IsGameCube ? DiscType.GameCube : DiscType.Wii; results.InputFileName = sourceFile.AllFiles.Length != 0 ? sourceFile.AllFiles[0] : sourceFile.FilePath; results.InputDiscNo = nstream.DiscNo; results.InputDiscVersion = nstream.Version; results.InputTitle = nstream.Title; results.InputId8 = nstream.Id8; results.InputSize = sourceSize; results.FullSize = nstream.ImageSize; results.Passes = passesText; if (pro.IsVerify) { results.OutputSize = results.InputSize; } } else if (pro.Writer is HashWriter) //hash writer gives no meaningful info back other than md5 and sha1 (the crc is forced when nkit, so ignore) { results.OutputMd5 = nr.OutputMd5; results.OutputSha1 = nr.OutputSha1; } else { if (nr.AliasJunkId != null) { results.AliasJunkId = nr.AliasJunkId; } if (nr.OutputTitle != null) { results.OutputDiscNo = nr.OutputDiscNo; results.OutputDiscVersion = nr.OutputDiscVersion; results.OutputTitle = nr.OutputTitle; } results.OutputId8 = nr.OutputId8; results.OutputCrc = nr.OutputCrc; results.OutputPrePatchCrc = nr.OutputPrePatchCrc; results.FullSize = nstream.ImageSize; if (tmp != null) { results.OutputSize = nr.OutputSize; } if (nr.ValidationCrc != 0 && results.VerifyCrc != 0) { results.VerifyCrc = 0; //blank verify if a new ValidationCrc is set - verification happened when both != 0 } if (nr.VerifyCrc != 0) { results.VerifyCrc = nr.VerifyCrc; } if (nr.ValidationCrc != 0) { results.ValidationCrc = nr.ValidationCrc; } if (nr.ValidateReadResult != VerifyResult.Unverified) { results.ValidateReadResult = nr.ValidateReadResult; } if (nr.VerifyOutputResult != VerifyResult.Unverified) { if (results.ValidateReadResult == VerifyResult.Unverified && nstream.IsNkit) { results.ValidateReadResult = nr.VerifyOutputResult; } results.VerifyOutputResult = nr.VerifyOutputResult; } if (nr.IsRecoverable) { results.IsRecoverable = nr.IsRecoverable; } } } finally { if (tmpFs != null) { tmpFs.Dispose(); } nstream.Close(); } if (lastTmp != null && tmp != null) { File.Delete(lastTmp); } //handle failures well if (results.ValidateReadResult == VerifyResult.VerifyFailed || results.VerifyOutputResult == VerifyResult.VerifyFailed) { lastTmp = tmp; //keep post loop happy break; } if (_completedPasses != _totalPasses) //last item { sf = SourceFiles.OpenFile(tmp ?? lastTmp); } if (tmp != null) { lastTmp = tmp; } } TimeSpan ts = DateTime.Now - dt; results.ProcessingTime = ts; //FAIL if (results.ValidateReadResult == VerifyResult.VerifyFailed || results.VerifyOutputResult == VerifyResult.VerifyFailed) { LogBlank(); Log(string.Format("Verification Failed Crc:{0} - Failed Test Crc:{1}", results.OutputCrc.ToString("X8"), results.ValidationCrc.ToString("X8"))); if (lastTmp != null) //only null when verify only { Log("Deleting Output" + (Settings.OutputLevel != 3 ? "" : " (Skipped as OutputLevel is 3:Debug)")); results.OutputFileName = null; if (Settings.OutputLevel != 3) { File.Delete(lastTmp); } LogBlank(); } } else //SUCCESS { LogBlank(); Log(string.Format("Completed ~ {0}m {1}s [MiB:{2:#.0}]", ((int)ts.TotalMinutes).ToString(), ts.Seconds.ToString(), (results.OutputSize / (double)(1024 * 1024)))); LogBlank(); Log("RESULTS"); Log("-------------------------------------------------------------------------------"); uint finalCrc = results.ValidationCrc != 0 ? results.ValidationCrc : results.OutputCrc; string mask = _context.Settings.MatchFailRenameToMask; results.OutputFileExt = "." + SourceFiles.ExtensionString(false, false, toNkit, nkitFormat == NkitFormatType.Gcz).ToLower(); results.RedumpInfo = _context.Dats.GetRedumpEntry(_context.Settings, _context.Dats, finalCrc); if (results.RedumpInfo.MatchType == MatchType.Redump || results.RedumpInfo.MatchType == MatchType.Custom) { Log(string.Format("{0} [{1} Name]", results.RedumpInfo.MatchName, results.RedumpInfo.MatchType.ToString())); if (results.IsRecoverable) { Log(string.Format("Missing Recovery data is required to correctly restore this image!", results.RedumpInfo.MatchName, results.RedumpInfo.MatchType.ToString())); } mask = results.RedumpInfo.MatchType == MatchType.Custom ? _context.Settings.CustomMatchRenameToMask : _context.Settings.RedumpMatchRenameToMask; } else { Log(string.Format("CRC {0} not in Redump or Custom Dat", finalCrc.ToString("X8"))); } LogBlank(); outputResults(results); if (lastTmp != null) //only null when verify only { if (testMode) { Log("TestMode: Deleting Output"); results.OutputFileName = null; if (File.Exists(lastTmp)) { File.Delete(lastTmp); } } else if (isRecovery && _context.Settings.RecoveryMatchFailDelete && results.RedumpInfo.MatchType == MatchType.MatchFail) { Log("Failed to Recover to Dat Entry: Deleting Output"); results.OutputFileName = null; File.Delete(lastTmp); } else { if (renameWithMasks) { results.OutputFileName = _context.Dats.GetFilename(results, mask); Log("Renaming Output Using Masks"); } else { results.OutputFileName = SourceFiles.GetUniqueName(sourceFile.CreateOutputFilename(results.OutputFileExt)); Log("Renaming Output Based on Source File" + (sourceFile.AllFiles.Count() > 1 ? "s" : "")); } LogBlank(); string path = Path.GetDirectoryName(results.OutputFileName); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } File.Move(lastTmp, results.OutputFileName); Log(string.Format("Output: {0}", Path.GetDirectoryName(results.OutputFileName))); Log(string.Format(" {0}", Path.GetFileName(results.OutputFileName))); //double check test mode just to be sure if (_context.Settings.DeleteSource && !testMode && results.VerifyOutputResult == VerifyResult.VerifySuccess) { LogBlank(); Log("Deleting Source:"); foreach (string s in sourceFile.AllFiles.Length == 0 ? new string[] { sourceFile.FilePath } : sourceFile.AllFiles) { Log(string.Format(" {0}", s)); File.Delete(s); } } } LogBlank(); } } } catch (Exception ex) { try { if (lastTmp == null) { lastTmp = tmp; } if (lastTmp != null) { LogBlank(); Log("Deleting Output" + (Settings.OutputLevel != 3 ? "" : " (Skipped as OutputLevel is 3:Debug)")); if (results != null) { results.OutputFileName = null; } if (Settings.OutputLevel != 3) { File.Delete(lastTmp); } } } catch { } if (_context.Settings.EnableSummaryLog) { if (results == null) { results = new OutputResults(); results.Conversion = this.ConvertionName; results.DiscType = (nstream?.IsGameCube ?? true) ? DiscType.GameCube : DiscType.Wii; results.InputFileName = (sourceFile?.AllFiles?.Length ?? 0) == 0 ? (sourceFile?.FilePath ?? "") : (sourceFile?.AllFiles.FirstOrDefault() ?? ""); results.InputDiscNo = nstream?.DiscNo ?? 0; results.InputDiscVersion = nstream?.Version ?? 0; results.InputTitle = nstream?.Title ?? ""; results.InputId8 = nstream?.Id8 ?? ""; results.InputSize = sourceFile?.Length ?? 0; } results.VerifyOutputResult = VerifyResult.Error; HandledException hex = ex as HandledException; if (hex == null) { hex = new HandledException(ex, "Unhandled Exception"); } results.ErrorMessage = hex.FriendlyErrorMessage; } throw; } finally { if (_context.Settings.EnableSummaryLog) { summaryLog(_context.Settings, results); Log("Summary Log Written" + (results.VerifyOutputResult != VerifyResult.Error ? "" : " as Errored!")); LogBlank(); } } return(results); }