private static void ClearResults(LogParseState state) { void Copy(params string[] keys) { foreach (var key in keys) { if (state.CompleteCollection?[key] is string value) { state.WipCollection[key] = value; } } } state.WipCollection = new NameValueCollection(); Copy( "build_and_specs", "fw_version_installed", "vulkan_gpu", "d3d_gpu", "driver_version", "driver_manuf", "driver_manuf_new", "driver_version_new", "vulkan_found_device", "vulkan_compatible_device_name", "vulkan_gpu", "vulkan_driver_version_raw", "compat_database_path" ); #if DEBUG Console.WriteLine("===== cleared"); #endif }
private static void ClearResults(LogParseState state) { void Copy(params string[] keys) { foreach (var key in keys) { if (state.CompleteCollection?[key] is string value) { state.WipCollection[key] = value; } if (state.CompleteMultiValueCollection?[key] is UniqueList <string> collection) { state.WipMultiValueCollection[key] = collection; } } } state.WipCollection = new NameValueCollection(); state.WipMultiValueCollection = new NameUniqueObjectCollection <string>(); Copy( "build_and_specs", "fw_version_installed", "vulkan_gpu", "d3d_gpu", "driver_version", "driver_manuf", "driver_manuf_new", "driver_version_new", "vulkan_found_device", "vulkan_compatible_device_name", "vulkan_gpu", "vulkan_driver_version_raw", "compat_database_path" ); Config.Log.Trace("===== cleared"); }
private static void OnExtractorHit(string buffer, Regex extractor, LogParseState state) { var matches = extractor.Matches(buffer); foreach (Match match in matches) { foreach (Group group in match.Groups) { if (!string.IsNullOrEmpty(group.Name) && group.Name != "0" && !string.IsNullOrWhiteSpace(group.Value)) { Config.Log.Debug($"regex {group.Name} = {group.Value}"); if (MultiValueItems.Contains(group.Name)) { var currentValue = state.WipCollection[group.Name]; if (!string.IsNullOrEmpty(currentValue)) { currentValue += Environment.NewLine; } state.WipCollection[group.Name] = currentValue + group.Value.ToUtf8(); } else { state.WipCollection[group.Name] = group.Value.ToUtf8(); } } } } }
private static void MarkAsComplete(LogParseState state) { state.CompleteCollection = state.WipCollection; #if DEBUG Console.WriteLine("----- complete section"); #endif }
private static async Task PiracyCheckAsync(string line, LogParseState state) { if (await PiracyStringProvider.FindTriggerAsync(line).ConfigureAwait(false) is string match) { state.PiracyTrigger = match; state.PiracyContext = line.ToUtf8(); state.Error = LogParseState.ErrorCode.PiracyDetected; } }
public static async Task <DiscordEmbedBuilder> AsEmbedAsync(this LogParseState state, DiscordClient client, DiscordMessage message, ISource source) { DiscordEmbedBuilder builder; var collection = state.CompleteCollection ?? state.WipCollection; if (collection?.Count > 0) { if (collection["serial"] is string serial && KnownDiscOnPsnIds.TryGetValue(serial, out var psnSerial) && collection["ldr_game_serial"] is string ldrGameSerial && ldrGameSerial.StartsWith("NP", StringComparison.InvariantCultureIgnoreCase) && ldrGameSerial.Equals(psnSerial, StringComparison.InvariantCultureIgnoreCase)) { collection["serial"] = psnSerial; collection["game_category"] = "HG"; } var gameInfo = await client.LookupGameInfoAsync(collection["serial"], collection["game_title"], true).ConfigureAwait(false); builder = new DiscordEmbedBuilder(gameInfo) { ThumbnailUrl = null }; // or this will f**k up all formatting collection["embed_title"] = builder.Title ?? ""; if (state.Error == LogParseState.ErrorCode.PiracyDetected) { var msg = "__You are being denied further support until you legally dump the game__.\n" + "Please note that the RPCS3 community and its developers do not support piracy.\n" + "Most of the issues with pirated dumps occur due to them having been tampered with in some way " + "and therefore act unpredictably on RPCS3.\n" + "If you need help obtaining legal dumps, please read [the quickstart guide](https://rpcs3.net/quickstart)."; builder.WithColor(Config.Colors.LogAlert) .WithTitle("Pirated content detected") .WithDescription(msg); } else { CleanupValues(collection); BuildInfoSection(builder, collection); var colA = BuildCpuSection(collection); var colB = BuildGpuSection(collection); BuildSettingsSections(builder, collection, colA, colB); BuildLibsSection(builder, collection); await BuildNotesSectionAsync(builder, state, collection, client).ConfigureAwait(false); } } else { builder = new DiscordEmbedBuilder { Description = "Log analysis failed, most likely cause is an empty log. Please try again.", Color = Config.Colors.LogResultFailed, }; } builder.AddAuthor(client, message, source, state); return(builder); }
private static Task LimitedPiracyCheckAsync(string line, LogParseState state) { if (state.LinesAfterConfig > 10) { return(Task.CompletedTask); } state.LinesAfterConfig++; return(PiracyCheckAsync(line, state)); }
private static async Task PiracyCheckAsync(string line, LogParseState state) { if (await ContentFilter.FindTriggerAsync(FilterContext.Log, line).ConfigureAwait(false) is Piracystring match && match.Actions.HasFlag(FilterAction.RemoveContent)) { var m = match; if (line.Contains("not valid, removing from") || line.Contains("Invalid disc path registered")) { m = new Piracystring { Id = match.Id, Actions = match.Actions & ~FilterAction.IssueWarning, Context = match.Context, CustomMessage = match.CustomMessage, Disabled = match.Disabled, ExplainTerm = match.ExplainTerm, String = match.String, ValidatingRegex = match.ValidatingRegex, } } ; if (state.FilterTriggers.TryGetValue(m.Id, out var fh)) { var updatedActions = fh.filter.Actions | m.Actions; if (fh.context.Length > line.Length) { m.Actions = updatedActions; state.FilterTriggers[m.Id] = (m, line.ToUtf8()); } else { fh.filter.Actions = updatedActions; } if (updatedActions.HasFlag(FilterAction.IssueWarning)) { state.Error = LogParseState.ErrorCode.PiracyDetected; } } else { var utf8line = line.ToUtf8(); state.FilterTriggers[m.Id] = (m, utf8line); if (m.Actions.HasFlag(FilterAction.IssueWarning)) { state.Error = LogParseState.ErrorCode.PiracyDetected; } } } }
public static async Task <DiscordEmbed> AsEmbedAsync(this LogParseState state, DiscordClient client, DiscordMessage message) { DiscordEmbedBuilder builder; var collection = state.CompleteCollection ?? state.WipCollection; if (collection?.Count > 0) { var gameInfo = await client.LookupGameInfoAsync(collection["serial"], collection["game_title"], true).ConfigureAwait(false); builder = new DiscordEmbedBuilder(gameInfo) { ThumbnailUrl = null }; // or this will f**k up all formatting if (state.Error == LogParseState.ErrorCode.PiracyDetected) { state.PiracyContext = state.PiracyContext.Sanitize(); var msg = "__You are being denied further support until you legally dump the game__.\n" + "Please note that the RPCS3 community and its developers do not support piracy.\n" + "Most of the issues with pirated dumps occur due to them having been tampered with in some way " + "and therefore act unpredictably on RPCS3.\n" + "If you need help obtaining legal dumps, please read [the quickstart guide](https://rpcs3.net/quickstart)."; builder.WithColor(Config.Colors.LogAlert) .WithTitle("Pirated release detected") .WithDescription(msg); } else { CleanupValues(collection); BuildInfoSection(builder, collection); var colA = BuildCpuSection(collection); var colB = BuildGpuSection(collection); BuildSettingsSections(builder, collection, colA, colB); BuildLibsSection(builder, collection); await BuildNotesSectionAsync(builder, state, collection, client).ConfigureAwait(false); } } else { builder = new DiscordEmbedBuilder { Description = "Log analysis failed, most likely cause is an empty log. Please try again.", Color = Config.Colors.LogResultFailed, }; } return(builder.Build()); }
private static void ClearResults(LogParseState state) { void Copy(params string[] keys) { foreach (var key in keys) { if (state.CompleteCollection?[key] is string value) { state.WipCollection[key] = value; } } } state.WipCollection = new NameValueCollection(); Copy( "build_and_specs", "vulkan_gpu", "d3d_gpu", "driver_version", "driver_manuf", "driver_manuf_new", "driver_version_new" ); }
private static async Task BuildNotesSectionAsync(DiscordEmbedBuilder builder, LogParseState state, DiscordClient discordClient) { var items = state.CompleteCollection; var multiItems = state.CompleteMultiValueCollection; var notes = new List <string>(); var(_, brokenDump, longestPath) = await HasBrokenFilesAsync(state).ConfigureAwait(false); brokenDump |= multiItems["edat_block_offset"].Any(); var supportedGpu = string.IsNullOrEmpty(items["rsx_unsupported_gpu"]); var unsupportedGpuDriver = false; var elfBootPath = items["elf_boot_path"] ?? ""; var isEboot = !string.IsNullOrEmpty(elfBootPath) && elfBootPath.EndsWith("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase); var isElf = !string.IsNullOrEmpty(elfBootPath) && !elfBootPath.EndsWith("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase); var serial = items["serial"] ?? ""; if (multiItems["fatal_error"] is UniqueList <string> fatalErrors && fatalErrors.Any()) { var contexts = multiItems["fatal_error_context"]; var reducedFatalErros = GroupSimilar(fatalErrors); foreach (var(fatalError, count, similarity) in reducedFatalErros) { var knownFatal = false; if (fatalError.Contains("psf.cpp", StringComparison.InvariantCultureIgnoreCase) || fatalError.Contains("invalid map<K, T>", StringComparison.InvariantCultureIgnoreCase) || contexts.Any(c => c.Contains("SaveData", StringComparison.InvariantCultureIgnoreCase))) { knownFatal = true; notes.Add("❌ Game save data is corrupted"); } else if (fatalError.Contains("Could not bind OpenGL context")) { knownFatal = true; notes.Add("❌ GPU or installed GPU drivers do not support OpenGL 4.3"); } else if (fatalError.Contains("file is null")) { if (contexts.Any(c => c.StartsWith("RSX"))) { knownFatal = true; notes.Add("❌ Shader cache might be corrupted; right-click on the game, then `Remove` → `Shader Cache`"); } if (contexts.Any(c => c.StartsWith("SPU"))) { knownFatal = true; notes.Add("❌ SPU cache might be corrupted; right-click on the game, then `Remove` → `SPU Cache`"); } if (contexts.Any(c => c.StartsWith("PPU"))) { knownFatal = true; notes.Add("❌ PPU cache might be corrupted; right-click on the game, then `Remove` → `PPU Cache`"); } } else if (fatalError.Contains("Null function") && fatalError.Contains("JIT")) { if (contexts.Any(c => c.StartsWith("PPU"))) { knownFatal = true; notes.Add("❌ PPU cache has issues; right-click on the game, then `Remove` → `PPU Cache`"); } if (contexts.Any(c => c.StartsWith("SPU"))) { knownFatal = true; notes.Add("❌ SPU cache has issues; right-click on the game, then `Remove` → `SPU Cache`"); } } else if (fatalError.Contains("no matching overloaded function found")) { if (fatalError.Contains("'mov'")) { knownFatal = true; unsupportedGpuDriver = true; } } else if (fatalError.Contains("RSX Decompiler Thread")) { if (items["build_branch"]?.ToLowerInvariant() == "head" && Version.TryParse(items["build_full_version"], out var v) && v >= decompilerIssueStartVersion && v < decompilerIssueEndVersion) { knownFatal = true; notes.Add("❌ This RPCS3 build has a known regression, please update to the latest version"); } } else if (fatalError.Contains("graphics-hook64.dll")) { knownFatal = true; notes.Add("❌ Please update or uninstall OBS to prevent crashes"); } else if (fatalError.Contains("bdcamvk64.dll")) { knownFatal = true; notes.Add("❌ Please update or uninstall Bandicam to prevent crashes"); } else if (fatalError.Contains("(e=0x17): file::read")) { // on windows this is ERROR_CRC notes.Add("❌ Storage device communication error; check your cables"); } else if (fatalError.Contains("Unknown primitive type")) { notes.Add("⚠ RSX desync detected, it's probably random"); } if (!knownFatal) { var sectionName = count == 1 ? "Fatal Error" #if DEBUG : $"Fatal Error (x{count}) [{similarity*100:0.00}%+]";
private static async Task BuildNotesSectionAsync(DiscordEmbedBuilder builder, LogParseState state, NameValueCollection items, DiscordClient discordClient) { var notes = new List <string>(); BuildWeirdSettingsSection(builder, items, notes); BuildMissingLicensesSection(builder, items); var(irdChecked, brokenDump, longestPath) = await HasBrokenFilesAsync(items).ConfigureAwait(false); brokenDump |= !string.IsNullOrEmpty(items["edat_block_offset"]); var elfBootPath = items["elf_boot_path"] ?? ""; var isEboot = !string.IsNullOrEmpty(elfBootPath) && elfBootPath.EndsWith("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase); var isElf = !string.IsNullOrEmpty(elfBootPath) && !elfBootPath.EndsWith("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase); var serial = items["serial"] ?? ""; if (items["fatal_error"] is string fatalError) { var context = items["fatal_error_context"] ?? ""; builder.AddField("Fatal Error", $"```\n{fatalError.Trim(1020)}\n```"); if (fatalError.Contains("psf.cpp", StringComparison.InvariantCultureIgnoreCase) || fatalError.Contains("invalid map<K, T>", StringComparison.InvariantCultureIgnoreCase) || context.Contains("SaveData", StringComparison.InvariantCultureIgnoreCase)) { notes.Add("❌ Game save data is corrupted"); } else if (fatalError.Contains("Could not bind OpenGL context")) { notes.Add("❌ GPU or installed GPU drivers do not support OpenGL 4.3"); } else if (fatalError.Contains("file is null")) { if (context.StartsWith("RSX", StringComparison.InvariantCultureIgnoreCase) || fatalError.StartsWith("RSX:")) { notes.Add("❌ Shader cache might be corrupted; right-click on the game, then `Remove` → `Shader Cache`"); } if (context.StartsWith("SPU", StringComparison.InvariantCultureIgnoreCase)) { notes.Add("❌ SPU cache might be corrupted; right-click on the game, then `Remove` → `SPU Cache`"); } if (context.StartsWith("PPU", StringComparison.InvariantCultureIgnoreCase)) { notes.Add("❌ PPU cache might be corrupted; right-click on the game, then `Remove` → `PPU Cache`"); } } else if (fatalError.Contains("(e=0x17): file::read")) { // on windows this is ERROR_CRC notes.Add("❌ Storage device communication error; check your cables"); } else if (fatalError.Contains("Unknown primitive type")) { notes.Add("⚠ RSX desync detected, it's probably random"); } } else if (items["unimplemented_syscall"] is string unimplementedSyscall) { if (unimplementedSyscall.Contains("syscall_988")) { fatalError = "Unimplemented syscall " + unimplementedSyscall; builder.AddField("Fatal Error", $"```{fatalError.Trim(1022)}```"); if (items["ppu_decoder"] is string ppuDecoder && ppuDecoder.Contains("Recompiler") && !Config.Colors.CompatStatusPlayable.Equals(builder.Color.Value)) { notes.Add("⚠ PPU desync detected; check your save data for corruption and/or try PPU Interpreter"); } else { notes.Add("⚠ PPU desync detected, most likely cause is corrupted save data"); } } }
private static async Task OnNewLineAsync(ReadOnlySequence <byte> line, ReadOnlySequence <byte> buffer, LinkedList <ReadOnlySequence <byte> > sectionLines, LogParseState state) { var currentProcessor = SectionParsers[state.Id]; if (line.AsString().Contains(currentProcessor.EndTrigger, StringComparison.InvariantCultureIgnoreCase)) { await FlushAllLinesAsync(buffer, sectionLines, state).ConfigureAwait(false); SectionParsers[state.Id].OnSectionEnd?.Invoke(state); state.Id++; return; } if (sectionLines.Count == 50) { await ProcessFirstLineInBufferAsync(buffer, sectionLines, state).ConfigureAwait(false); } sectionLines.AddLast(line); }
public static async Task <LogParseState> ReadPipeAsync(PipeReader reader) { var timeout = new CancellationTokenSource(Config.LogParsingTimeout); var currentSectionLines = new LinkedList <ReadOnlySequence <byte> >(); var state = new LogParseState(); bool skippedBom = false; long totalReadBytes = 0; ReadResult result; do { result = await reader.ReadAsync(Config.Cts.Token).ConfigureAwait(false); var buffer = result.Buffer; if (!skippedBom) { if (buffer.Length < 3) { continue; } var potentialBom = buffer.Slice(0, 3); if (potentialBom.ToArray().SequenceEqual(Bom)) { reader.AdvanceTo(potentialBom.End); totalReadBytes += potentialBom.Length; skippedBom = true; continue; } skippedBom = true; } SequencePosition?lineEnd; do { if (currentSectionLines.Count > 0) { buffer = buffer.Slice(buffer.GetPosition(1, currentSectionLines.Last.Value.End)); } lineEnd = buffer.PositionOf((byte)'\n'); if (lineEnd != null) { await OnNewLineAsync(buffer.Slice(0, lineEnd.Value), result.Buffer, currentSectionLines, state).ConfigureAwait(false); if (state.Error != LogParseState.ErrorCode.None) { reader.Complete(); return(state); } buffer = buffer.Slice(buffer.GetPosition(1, lineEnd.Value)); } } while (lineEnd != null); if (result.IsCanceled || Config.Cts.IsCancellationRequested || timeout.IsCancellationRequested) { state.Error = LogParseState.ErrorCode.SizeLimit; } else if (result.IsCompleted) { await FlushAllLinesAsync(result.Buffer, currentSectionLines, state).ConfigureAwait(false); } var sectionStart = currentSectionLines.Count == 0 ? buffer : currentSectionLines.First.Value; totalReadBytes += result.Buffer.Slice(0, sectionStart.Start).Length; reader.AdvanceTo(sectionStart.Start); if (totalReadBytes >= Config.LogSizeLimit) { state.Error = LogParseState.ErrorCode.SizeLimit; break; } } while (!(result.IsCompleted || result.IsCanceled || Config.Cts.IsCancellationRequested || timeout.IsCancellationRequested)); reader.Complete(); return(state); }
private static async Task ProcessFirstLineInBufferAsync(ReadOnlySequence <byte> buffer, LinkedList <ReadOnlySequence <byte> > sectionLines, LogParseState state) { var currentProcessor = SectionParsers[state.Id]; var firstSectionLine = sectionLines.First.Value.AsString(); await currentProcessor.OnLineCheckAsync(firstSectionLine, state).ConfigureAwait(false); if (state.Error != LogParseState.ErrorCode.None) { return; } var section = buffer.Slice(sectionLines.First.Value.Start, sectionLines.Last.Value.End).AsString(); currentProcessor.OnExtract?.Invoke(firstSectionLine, section, state); sectionLines.RemoveFirst(); }
public static async Task <LogParseState> ParseLogAsync(ISource source, Func <Task> onProgressAsync, CancellationToken cancellationToken) { LogParseState result = null; try { var pipe = new Pipe(); var fillPipeTask = source.FillPipeAsync(pipe.Writer, cancellationToken); var readPipeTask = LogParser.ReadPipeAsync(pipe.Reader, cancellationToken); do { await Task.WhenAny(readPipeTask, Task.Delay(5000, cancellationToken)).ConfigureAwait(false); if (!readPipeTask.IsCompleted) { await onProgressAsync().ConfigureAwait(false); } } while (!readPipeTask.IsCompleted && !cancellationToken.IsCancellationRequested); result = await readPipeTask.ConfigureAwait(false); await fillPipeTask.ConfigureAwait(false); result.TotalBytes = source.LogFileSize; if (result.FilterTriggers.Any()) { var(f, c) = result.FilterTriggers.Values.FirstOrDefault(ft => ft.filter.Actions.HasFlag(FilterAction.IssueWarning)); if (f == null) { (f, c) = result.FilterTriggers.Values.FirstOrDefault(ft => ft.filter.Actions.HasFlag(FilterAction.RemoveContent)); } if (f == null) { (f, c) = result.FilterTriggers.Values.FirstOrDefault(); } result.SelectedFilter = f; result.SelectedFilterContext = c; } #if DEBUG Config.Log.Debug("~~~~~~~~~~~~~~~~~~~~"); Config.Log.Debug("Extractor hit stats:"); foreach (var stat in result.ExtractorHitStats.OrderByDescending(kvp => kvp.Value)) { if (stat.Value > 100000) { Config.Log.Fatal($"{stat.Value}: {stat.Key}"); } else if (stat.Value > 10000) { Config.Log.Error($"{stat.Value}: {stat.Key}"); } else if (stat.Value > 1000) { Config.Log.Warn($"{stat.Value}: {stat.Key}"); } else if (stat.Value > 100) { Config.Log.Info($"{stat.Value}: {stat.Key}"); } else { Config.Log.Debug($"{stat.Value}: {stat.Key}"); } } Config.Log.Debug("~~~~~~~~~~~~~~~~~~~~"); Config.Log.Debug("Syscall stats:"); int serialCount = result.Syscalls.Count, moduleCount = 0, functionCount = 0; foreach (var moduleStats in result.Syscalls.Values) { moduleCount += moduleStats.Count; foreach (var funcStats in moduleStats.Values) { functionCount += funcStats.Count; } } Config.Log.Debug("Product keys: " + serialCount); Config.Log.Debug("Modules: " + moduleCount); Config.Log.Debug("Functions: " + functionCount); Config.Log.Debug("Saving syscall information..."); var sw = Stopwatch.StartNew(); #endif await SyscallInfoProvider.SaveAsync(result.Syscalls).ConfigureAwait(false); #if DEBUG Config.Log.Debug("Saving syscall information took " + sw.Elapsed); #endif } catch (Exception e) { Config.Log.Error(e, "Log parsing failed"); } return(result); }
private static void OnExtractorHit(string buffer, string trigger, Regex extractor, LogParseState state) { if (trigger == "{PPU[") { if (state.WipCollection["serial"] is string serial) { var match = extractor.Match(buffer); if (match.Success && match.Groups["syscall_module"]?.Value.ToUtf8() is string syscallModule && match.Groups["syscall_name"]?.Value.ToUtf8() is string syscallName) { lock (state) { if (!state.Syscalls.TryGetValue(serial, out var serialSyscallStats)) { state.Syscalls[serial] = serialSyscallStats = new Dictionary <string, HashSet <string> >(); } if (!serialSyscallStats.TryGetValue(syscallModule, out var moduleStats)) { serialSyscallStats[syscallModule] = moduleStats = new HashSet <string>(); } moduleStats.Add(syscallName); } } } } else { var matches = extractor.Matches(buffer); if (matches.Count == 0) { return; } foreach (Match match in matches) { foreach (Group group in match.Groups) { if (!string.IsNullOrEmpty(group.Name) && group.Name != "0" && !string.IsNullOrWhiteSpace(group.Value)) { if (string.IsNullOrEmpty(group.Value)) { continue; } var strValue = group.Value.ToUtf8(); Config.Log.Trace($"regex {group.Name} = {group.Value}"); lock (state) { if (MultiValueItems.Contains(group.Name)) { var currentValue = state.WipCollection[group.Name]; if (!string.IsNullOrEmpty(currentValue)) { currentValue += Environment.NewLine; } state.WipCollection[group.Name] = currentValue + strValue; } else { state.WipCollection[group.Name] = strValue; } if (CountValueItems.Contains(group.Name)) { state.ValueHitStats.TryGetValue(group.Name, out var hits); state.ValueHitStats[group.Name] = ++hits; } } } } } } }
private static void MarkAsCompleteAndReset(LogParseState state) { MarkAsComplete(state); ClearResults(state); state.Id = -1; }
private static void MarkAsComplete(LogParseState state) { state.CompleteCollection = state.WipCollection; }
internal static DiscordEmbedBuilder AddAuthor(this DiscordEmbedBuilder builder, DiscordClient client, DiscordMessage message, ISource source, LogParseState state = null) { if (state?.Error == LogParseState.ErrorCode.PiracyDetected) { return(builder); } if (message != null) { var author = message.Author; var member = client.GetMember(message.Channel?.Guild, author); string msg; if (member == null) { msg = $"Log from {author.Username.Sanitize()} | {author.Id}\n"; } else { msg = $"Log from {member.DisplayName.Sanitize()} | {member.Id}\n"; } msg += " | " + (source?.SourceType ?? "Unknown source"); if (state?.ReadBytes > 0 && source?.LogFileSize > 0 && source.LogFileSize < 2L * 1024 * 1024 * 1024 && state.ReadBytes <= source.LogFileSize) { msg += $" | Parsed {state.ReadBytes * 100.0 / source.LogFileSize:0.##}%"; } else if (source?.SourceFilePosition > 0 && source.SourceFileSize > 0 && source.SourceFilePosition <= source.SourceFileSize) { msg += $" | Read {source.SourceFilePosition * 100.0 / source.SourceFileSize:0.##}%"; } else if (state?.ReadBytes > 0) { msg += $" | Parsed {state.ReadBytes} byte{(state.ReadBytes == 1 ? "" : "s")}"; } else if (source?.LogFileSize > 0) { msg += $" | {source.LogFileSize} byte{(source.LogFileSize == 1 ? "" : "s")}"; } #if DEBUG if (state?.ParsingTime.TotalMilliseconds > 0) { msg += $" | {state.ParsingTime.TotalSeconds:0.###}s"; } msg += " | Test Bot Instance"; #endif builder.WithFooter(msg); } return(builder); }
private static async Task OnNewLineAsync(ReadOnlySequence <byte> line, ReadOnlySequence <byte> buffer, LinkedList <ReadOnlySequence <byte> > sectionLines, LogParseState state) { var currentProcessor = SectionParsers[state.Id]; var strLine = line.AsString(); if (currentProcessor.EndTrigger.Any(et => strLine.Contains(et))) { await FlushAllLinesAsync(buffer, sectionLines, state).ConfigureAwait(false); await TaskScheduler.WaitForClearTagAsync(state).ConfigureAwait(false); SectionParsers[state.Id].OnSectionEnd?.Invoke(state); state.Id++; } if (sectionLines.Count == 50) { await ProcessFirstLineInBufferAsync(buffer, sectionLines, state).ConfigureAwait(false); } sectionLines.AddLast(line); }
public static async Task <LogParseState> ReadPipeAsync(PipeReader reader, CancellationToken cancellationToken) { var currentSectionLines = new LinkedList <ReadOnlySequence <byte> >(); var state = new LogParseState(); bool skippedBom = false; long totalReadBytes = 0; ReadResult result; do { try { result = await reader.ReadAsync(cancellationToken).ConfigureAwait(false); var buffer = result.Buffer; if (!skippedBom) { if (buffer.Length < 3) { continue; } var potentialBom = buffer.Slice(0, 3); if (potentialBom.ToArray().SequenceEqual(Bom)) { reader.AdvanceTo(potentialBom.End); totalReadBytes += potentialBom.Length; skippedBom = true; continue; } skippedBom = true; } SequencePosition?lineEnd; do { if (currentSectionLines.Count > 0) { buffer = buffer.Slice(buffer.GetPosition(1, currentSectionLines.Last.Value.End)); } lineEnd = buffer.PositionOf((byte)'\n'); if (lineEnd != null) { await OnNewLineAsync(buffer.Slice(0, lineEnd.Value), result.Buffer, currentSectionLines, state).ConfigureAwait(false); if (state.Error != LogParseState.ErrorCode.None) { reader.Complete(); return(state); } buffer = buffer.Slice(buffer.GetPosition(1, lineEnd.Value)); } } while (lineEnd != null); if (result.IsCanceled || cancellationToken.IsCancellationRequested) { if (state.Error == LogParseState.ErrorCode.None) { state.Error = LogParseState.ErrorCode.SizeLimit; } } else if (result.IsCompleted) { if (!buffer.End.Equals(currentSectionLines.Last.Value.End)) { await OnNewLineAsync(buffer.Slice(0), result.Buffer, currentSectionLines, state).ConfigureAwait(false); } await FlushAllLinesAsync(result.Buffer, currentSectionLines, state).ConfigureAwait(false); } var sectionStart = currentSectionLines.Count == 0 ? buffer : currentSectionLines.First.Value; totalReadBytes += result.Buffer.Slice(0, sectionStart.Start).Length; reader.AdvanceTo(sectionStart.Start); } catch (Exception e) { Config.Log.Warn(e, "Aborted log parsing due to exception"); if (state.Error == LogParseState.ErrorCode.None) { state.Error = LogParseState.ErrorCode.UnknownError; } break; } } while (!(result.IsCompleted || result.IsCanceled || cancellationToken.IsCancellationRequested)); await TaskScheduler.WaitForClearTagAsync(state).ConfigureAwait(false); state.ReadBytes = totalReadBytes; reader.Complete(); return(state); }
private static void MarkAsComplete(LogParseState state) { state.CompleteCollection = state.WipCollection; state.CompleteMultiValueCollection = state.WipMultiValueCollection; Config.Log.Trace("----- complete section"); }
public static async void EnqueueLogProcessing(DiscordClient client, DiscordChannel channel, DiscordMessage message, DiscordMember requester = null, bool checkExternalLinks = false) { try { if (!QueueLimiter.Wait(0)) { await channel.SendMessageAsync("Log processing is rate limited, try again a bit later").ConfigureAwait(false); return; } bool parsedLog = false; var startTime = Stopwatch.StartNew(); DiscordMessage botMsg = null; try { var possibleHandlers = sourceHandlers.Select(h => h.FindHandlerAsync(message, archiveHandlers).ConfigureAwait(false).GetAwaiter().GetResult()).ToList(); var source = possibleHandlers.FirstOrDefault(h => h.source != null).source; var fail = possibleHandlers.FirstOrDefault(h => !string.IsNullOrEmpty(h.failReason)).failReason; if (source != null) { Config.Log.Debug($">>>>>>> {message.Id % 100} Parsing log '{source.FileName}' from {message.Author.Username}#{message.Author.Discriminator} ({message.Author.Id}) using {source.GetType().Name} ({source.SourceFileSize} bytes)..."); var analyzingProgressEmbed = GetAnalyzingMsgEmbed(client); botMsg = await channel.SendMessageAsync(embed : analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false); parsedLog = true; LogParseState result = null; using (var timeout = new CancellationTokenSource(Config.LogParsingTimeout)) { using var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(timeout.Token, Config.Cts.Token); var tries = 0; do { result = await ParseLogAsync( source, async() => botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed : analyzingProgressEmbed.AddAuthor(client, message, source)).ConfigureAwait(false), combinedTokenSource.Token ).ConfigureAwait(false); tries++; } while (result == null && !combinedTokenSource.IsCancellationRequested && tries < 3); } if (result == null) { botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed : new DiscordEmbedBuilder { Description = "Log analysis failed, most likely cause is a truncated/invalid log.\n" + "Please run the game again and re-upload a new copy.", Color = Config.Colors.LogResultFailed, } .AddAuthor(client, message, source) .Build() ).ConfigureAwait(false); } else { result.ParsingTime = startTime.Elapsed; try { if (result.Error == LogParseState.ErrorCode.PiracyDetected) { var yarr = client.GetEmoji(":piratethink:", "☠"); result.ReadBytes = 0; if (message.Author.IsWhitelisted(client, channel.Guild)) { var piracyWarning = await result.AsEmbedAsync(client, message, source).ConfigureAwait(false); piracyWarning = piracyWarning.WithDescription("Please remove the log and issue warning to the original author of the log"); botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, embed : piracyWarning).ConfigureAwait(false); await client.ReportAsync(yarr + " Pirated Release (whitelisted by role)", message, result.SelectedFilter?.String, result.SelectedFilterContext, ReportSeverity.Low).ConfigureAwait(false); } else { var severity = ReportSeverity.Low; try { await message.DeleteAsync("Piracy detected in log").ConfigureAwait(false); } catch (Exception e) { severity = ReportSeverity.High; Config.Log.Warn(e, $"Unable to delete message in {channel.Name}"); } try { /* * botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, * $"{message.Author.Mention}, please read carefully:", * embed: await result.AsEmbedAsync(client, message, source).ConfigureAwait(false) * ).ConfigureAwait(false); */ botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, $"{message.Author.Mention}, please read carefully:\n" + "🏴☠️ **Pirated content detected** 🏴☠️\n" + "__You are being denied further support until you legally dump the game__.\n" + "Please note that the RPCS3 community and its developers do not support piracy.\n" + "Most of the issues with pirated dumps occur due to them being modified in some way " + "that prevent them from working on RPCS3.\n" + "If you need help obtaining valid working dump of the game you own, please read the quickstart guide at <https://rpcs3.net/quickstart>" ).ConfigureAwait(false); } catch (Exception e) { Config.Log.Error(e, "Failed to send piracy warning"); } try { await client.ReportAsync(yarr + " Pirated Release", message, result.SelectedFilter?.String, result.SelectedFilterContext, severity).ConfigureAwait(false); } catch (Exception e) { Config.Log.Error(e, "Failed to send piracy report"); } if (!(message.Channel.IsPrivate || (message.Channel.Name?.Contains("spam") ?? true))) { await Warnings.AddAsync(client, message, message.Author.Id, message.Author.Username, client.CurrentUser, "Pirated Release", $"{result.SelectedFilter?.String} - {result.SelectedFilterContext?.Sanitize()}"); } } } else { if (result.SelectedFilter != null) { await ContentFilter.PerformFilterActions(client, message, result.SelectedFilter, FilterAction.IssueWarning | FilterAction.SendMessage, result.SelectedFilterContext).ConfigureAwait(false); } botMsg = await botMsg.UpdateOrCreateMessageAsync(channel, requester == null?null : $"Analyzed log from {client.GetMember(channel.Guild, message.Author)?.GetUsernameWithNickname()} by request from {requester.Mention}:", embed : await result.AsEmbedAsync(client, message, source).ConfigureAwait(false) ).ConfigureAwait(false); } } catch (Exception e) { Config.Log.Error(e, "Sending log results failed"); } } return; } else if (!string.IsNullOrEmpty(fail) && ("help".Equals(channel.Name, StringComparison.InvariantCultureIgnoreCase) || LimitedToSpamChannel.IsSpamChannel(channel))) { await channel.SendMessageAsync($"{message.Author.Mention} {fail}").ConfigureAwait(false); return; } if (!"help".Equals(channel.Name, StringComparison.InvariantCultureIgnoreCase)) { return; } var potentialLogExtension = message.Attachments.Select(a => Path.GetExtension(a.FileName).ToUpperInvariant().TrimStart('.')).FirstOrDefault(); switch (potentialLogExtension) { case "TXT": { await channel.SendMessageAsync($"{message.Author.Mention} Please upload the full RPCS3.log.gz (or RPCS3.log with a zip/rar icon) file after closing the emulator instead of copying the logs from RPCS3's interface, as it doesn't contain all the required information.").ConfigureAwait(false); return; } } if (string.IsNullOrEmpty(message.Content)) { return; } var linkStart = message.Content.IndexOf("http"); if (linkStart > -1) { var link = message.Content[linkStart..].Split(linkSeparator, 2)[0]; if (link.Contains(".log", StringComparison.InvariantCultureIgnoreCase) || link.Contains("rpcs3.zip", StringComparison.CurrentCultureIgnoreCase)) { await channel.SendMessageAsync("If you intended to upload a log file please re-upload it directly to discord").ConfigureAwait(false); } }
private static void CleanupValues(LogParseState state) { var items = state.CompleteCollection; var multiItems = state.CompleteMultiValueCollection; if (items["strict_rendering_mode"] == "true") { items["resolution_scale"] = "Strict Mode"; } if (items["spu_threads"] == "0") { items["spu_threads"] = "Auto"; } if (items["spu_secondary_cores"] != null) { items["thread_scheduler"] = items["spu_secondary_cores"]; } if (items["vulkan_initialized_device"] != null) { items["gpu_info"] = items["vulkan_initialized_device"]; } else if (items["driver_manuf_new"] != null) { items["gpu_info"] = items["driver_manuf_new"]; } else if (items["vulkan_gpu"] != "\"\"") { items["gpu_info"] = items["vulkan_gpu"]; } else if (items["d3d_gpu"] != "\"\"") { items["gpu_info"] = items["d3d_gpu"]; } else if (items["driver_manuf"] != null) { items["gpu_info"] = items["driver_manuf"]; } if (!string.IsNullOrEmpty(items["gpu_info"])) { items["gpu_info"] = items["gpu_info"].StripMarks(); items["driver_version_info"] = GetVulkanDriverVersion(items["vulkan_initialized_device"], multiItems["vulkan_found_device"]) ?? GetVulkanDriverVersion(items["gpu_info"], multiItems["vulkan_found_device"]) ?? GetOpenglDriverVersion(items["gpu_info"], items["driver_version_new"] ?? items["driver_version"]) ?? GetVulkanDriverVersionRaw(items["gpu_info"], items["vulkan_driver_version_raw"]); } if (items["driver_version_info"] != null) { items["gpu_info"] += $" ({items["driver_version_info"]})"; } if (multiItems["vulkan_compatible_device_name"] is UniqueList <string> vulkanDevices && vulkanDevices.Any()) { var devices = vulkanDevices .Distinct() .Select(n => new { name = n.StripMarks(), driverVersion = GetVulkanDriverVersion(n, multiItems["vulkan_found_device"]) }) .Reverse() .ToList(); if (string.IsNullOrEmpty(items["gpu_info"]) && devices.Count > 0) { var discreteGpu = devices.FirstOrDefault(d => IsNvidia(d.name)) ?? devices.FirstOrDefault(d => IsAmd(d.name)) ?? devices.First(); items["discrete_gpu_info"] = $"{discreteGpu.name} ({discreteGpu.driverVersion})"; items["driver_version_info"] = discreteGpu.driverVersion; } items["gpu_available_info"] = string.Join(Environment.NewLine, devices.Select(d => $"{d.name} ({d.driverVersion})")); } if (items["af_override"] is string af) { if (af == "0") { items["af_override"] = "Auto"; } else if (af == "1") { items["af_override"] = "Disabled"; } } if (items["zcull"] == "true") { items["zcull_status"] = "Disabled"; } else if (items["relaxed_zcull"] == "true") { items["zcull_status"] = "Relaxed"; } else { items["zcull_status"] = "Full"; } if (items["lib_loader"] is string libLoader) { var liblv2 = libLoader.Contains("liblv2", StringComparison.InvariantCultureIgnoreCase); var auto = libLoader.Contains("auto", StringComparison.InvariantCultureIgnoreCase); var manual = libLoader.Contains("manual", StringComparison.InvariantCultureIgnoreCase); var strict = libLoader.Contains("strict", StringComparison.InvariantCultureIgnoreCase); if (auto && manual) { items["lib_loader"] = "Auto & manual select"; } else if (liblv2 && manual) { items["lib_loader"] = "Liblv2.sprx & manual"; } else if (liblv2 && strict) { items["lib_loader"] = "Liblv2.sprx & strict"; } else if (auto) { items["lib_loader"] = "Auto"; } else if (manual) { items["lib_loader"] = "Manual selection"; } else { items["lib_loader"] = "Liblv2.sprx only"; } }
private static async Task FlushAllLinesAsync(ReadOnlySequence <byte> buffer, LinkedList <ReadOnlySequence <byte> > sectionLines, LogParseState state) { while (sectionLines.Count > 0 && state.Error == LogParseState.ErrorCode.None) { await ProcessFirstLineInBufferAsync(buffer, sectionLines, state).ConfigureAwait(false); } }
private static async Task BuildNotesSectionAsync(DiscordEmbedBuilder builder, LogParseState state, DiscordClient discordClient) { var items = state.CompleteCollection; var multiItems = state.CompleteMultiValueCollection; var notes = new List <string>(); var(_, brokenDump, longestPath) = await HasBrokenFilesAsync(state).ConfigureAwait(false); brokenDump |= multiItems["edat_block_offset"].Any(); var elfBootPath = items["elf_boot_path"] ?? ""; var isEboot = !string.IsNullOrEmpty(elfBootPath) && elfBootPath.EndsWith("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase); var isElf = !string.IsNullOrEmpty(elfBootPath) && !elfBootPath.EndsWith("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase); var serial = items["serial"] ?? ""; if (multiItems["fatal_error"] is UniqueList <string> fatalErrors && fatalErrors.Any()) { var contexts = multiItems["fatal_error_context"]; var reducedFatalErros = GroupSimilar(fatalErrors); foreach (var(fatalError, count, similarity) in reducedFatalErros) { var knownFatal = false; if (fatalError.Contains("psf.cpp", StringComparison.InvariantCultureIgnoreCase) || fatalError.Contains("invalid map<K, T>", StringComparison.InvariantCultureIgnoreCase) || contexts.Any(c => c.Contains("SaveData", StringComparison.InvariantCultureIgnoreCase))) { knownFatal = true; notes.Add("❌ Game save data is corrupted"); } else if (fatalError.Contains("Could not bind OpenGL context")) { knownFatal = true; notes.Add("❌ GPU or installed GPU drivers do not support OpenGL 4.3"); } else if (fatalError.Contains("file is null")) { if (contexts.Any(c => c.StartsWith("RSX"))) { knownFatal = true; notes.Add("❌ Shader cache might be corrupted; right-click on the game, then `Remove` → `Shader Cache`"); } if (contexts.Any(c => c.StartsWith("SPU"))) { knownFatal = true; notes.Add("❌ SPU cache might be corrupted; right-click on the game, then `Remove` → `SPU Cache`"); } if (contexts.Any(c => c.StartsWith("PPU"))) { knownFatal = true; notes.Add("❌ PPU cache might be corrupted; right-click on the game, then `Remove` → `PPU Cache`"); } } else if (fatalError.Contains("Null function") && fatalError.Contains("JIT")) { if (contexts.Any(c => c.StartsWith("PPU"))) { knownFatal = true; notes.Add("❌ PPU cache has issues; right-click on the game, then `Remove` → `PPU Cache`"); } if (contexts.Any(c => c.StartsWith("SPU"))) { knownFatal = true; notes.Add("❌ SPU cache has issues; right-click on the game, then `Remove` → `SPU Cache`"); } } else if (fatalError.Contains("(e=0x17): file::read")) { // on windows this is ERROR_CRC notes.Add("❌ Storage device communication error; check your cables"); } else if (fatalError.Contains("Unknown primitive type")) { notes.Add("⚠ RSX desync detected, it's probably random"); } if (!knownFatal) { var sectionName = count == 1 ? "Fatal Error" #if DEBUG : $"Fatal Error (x{count} [{similarity*100:0.00}%+])";
public static async void BackgroundProcessor(MessageCreateEventArgs args) { try { var message = args.Message; if (!QueueLimiter.Wait(0)) { await args.Channel.SendMessageAsync("Log processing is rate limited, try again a bit later").ConfigureAwait(false); return; } bool parsedLog = false; var startTime = Stopwatch.StartNew(); try { foreach (var attachment in message.Attachments.Where(a => a.FileSize < Config.AttachmentSizeLimit && !a.FileName.EndsWith("tty.log", StringComparison.InvariantCultureIgnoreCase))) { foreach (var handler in handlers) { if (await handler.CanHandleAsync(attachment).ConfigureAwait(false)) { await args.Channel.TriggerTypingAsync().ConfigureAwait(false); Config.Log.Debug($">>>>>>> {message.Id % 100} Parsing log from attachment {attachment.FileName} ({attachment.FileSize})..."); parsedLog = true; LogParseState result = null; try { var pipe = new Pipe(); var fillPipeTask = handler.FillPipeAsync(attachment, pipe.Writer); result = await LogParser.ReadPipeAsync(pipe.Reader).ConfigureAwait(false); await fillPipeTask.ConfigureAwait(false); } catch (Exception e) { Config.Log.Error(e, "Log parsing failed"); } if (result == null) { await args.Channel.SendMessageAsync("Log analysis failed, most likely cause is a truncated/invalid log. Please run the game again and reupload the new copy.").ConfigureAwait(false); } else { try { if (result.Error == LogParseState.ErrorCode.PiracyDetected) { if (args.Author.IsWhitelisted(args.Client, args.Guild)) { await Task.WhenAll( args.Channel.SendMessageAsync("I see wha' ye did thar ☠"), args.Client.ReportAsync("Pirated Release (whitelisted by role)", args.Message, result.PiracyTrigger, result.PiracyContext, ReportSeverity.Low) ).ConfigureAwait(false); } else { var severity = ReportSeverity.Low; try { await message.DeleteAsync("Piracy detected in log").ConfigureAwait(false); } catch (Exception e) { severity = ReportSeverity.High; Config.Log.Warn(e, $"Unable to delete message in {args.Channel.Name}"); } await args.Channel.SendMessageAsync(embed : await result.AsEmbedAsync(args.Client, args.Message).ConfigureAwait(false)).ConfigureAwait(false); await Task.WhenAll( args.Client.ReportAsync("Pirated Release", args.Message, result.PiracyTrigger, result.PiracyContext, severity), Warnings.AddAsync(args.Client, args.Message, args.Message.Author.Id, args.Message.Author.Username, args.Client.CurrentUser, "Pirated Release", $"{message.Content.Sanitize()} - {result.PiracyTrigger}") ); } } else { await args.Channel.SendMessageAsync(embed : await result.AsEmbedAsync(args.Client, args.Message).ConfigureAwait(false)).ConfigureAwait(false); } } catch (Exception e) { Config.Log.Error(e, "Sending log results failed"); } } return; } } } if (!"help".Equals(args.Channel.Name, StringComparison.InvariantCultureIgnoreCase)) { return; } var potentialLogExtension = message.Attachments.Select(a => Path.GetExtension(a.FileName).ToUpperInvariant().TrimStart('.')).FirstOrDefault(); switch (potentialLogExtension) { case "TXT": { await args.Channel.SendMessageAsync($"{message.Author.Mention} please do not copy/paste logs from UI, they do not contain all the required information; ask if you do not know how to upload full log file").ConfigureAwait(false); return; } } if (string.IsNullOrEmpty(message.Content)) { return; } var linkStart = message.Content.IndexOf("http"); if (linkStart > -1) { var link = message.Content.Substring(linkStart).Split(linkSeparator, 2)[0]; if (link.Contains(".log", StringComparison.InvariantCultureIgnoreCase) || link.Contains("rpcs3.zip", StringComparison.CurrentCultureIgnoreCase)) { await args.Channel.SendMessageAsync("If you intended to upload a log file please re-upload it directly to discord").ConfigureAwait(false); } } } finally { QueueLimiter.Release(); if (parsedLog) { Config.Log.Debug($"<<<<<<< {message.Id % 100} Finished parsing in {startTime.Elapsed}"); } } } catch (Exception e) { Config.Log.Error(e, "Error parsing log"); } }