public void ProcessLogLine(LogLine logLine, LogType logType) { if (!(logLine.LineContents is string logString)) { _processingNotificationsCollector.ReportError("Failed to parse log line as string", logLine, nameof(SearchServerPlugin)); return; } var match = _regex.Match(logString); if (!match.Success) { _processingNotificationsCollector.ReportError("Failed to parse log line as SearchServer event", logLine, nameof(SearchServerPlugin)); return; } var parsed = new SearchServerEvent { Class = match.Groups["class"].Value, File = logLine.LogFileInfo.FileName, FilePath = logLine.LogFileInfo.FilePath, LineNumber = logLine.LineNumber, Message = match.Groups["message"].Value, Severity = match.Groups["sev"].Value, Timestamp = TimestampParsers.ParseJavaLogsTimestamp(match.Groups["ts"].Value), Worker = logLine.LogFileInfo.Worker, }; _writer.AddLine(parsed); }
private void ProcessCppLine(LogLine logLine) { if (!(logLine.LineContents is NativeJsonLogsBaseEvent baseEvent)) { _processingNotificationsCollector.ReportError($"Was not able to cast line contents as {nameof(NativeJsonLogsBaseEvent)}", logLine, nameof(DataServerPlugin)); return; } if (baseEvent.EventPayload == null) { _processingNotificationsCollector.ReportError("Log line does not contain event payload (\"v\" key in json). Skipping this line", logLine, nameof(DataServerPlugin)); return; } if (ShouldSkip(baseEvent)) { return; } try { var @event = new DataServerEvent(logLine, baseEvent); _writer.AddLine(@event); } catch (JsonException ex) { var message = $"Exception occurred while parsing log line. Most likely - something is wrong with JSON format. Event type: `{baseEvent?.EventType ?? "(null)"}`. Exception message: `{ex.Message}`"; _processingNotificationsCollector.ReportError(message, logLine, nameof(DataServerPlugin)); } }
private ReadLogLineResult TurnStringIntoNativeJsonLogBaseEvent(ReadLogLineResult originalEvent) { if (!(originalEvent.LineContent is string lineString)) { _processingNotificationsCollector.ReportError("Can't interpret line as string", _filePath, originalEvent.LineNumber, nameof(NativeJsonLogsReader)); return(new ReadLogLineResult(originalEvent.LineNumber, null)); } try { try { var deserializedObject = JsonConvert.DeserializeObject <NativeJsonLogsBaseEvent>(lineString, _serializerSettings); return(new ReadLogLineResult(originalEvent.LineNumber, deserializedObject)); } catch (JsonReaderException ex) when(ex.Message.StartsWith(@"Invalid JavaScript property identifier character:")) { // Check for and repair properties using an incorrect ',' character as a number decimal separator in unquoted properties var repairedLineString = NumberDecimalSeparatorRepairRegex.Replace(lineString, "$1.$2"); var deserializedObject = JsonConvert.DeserializeObject <NativeJsonLogsBaseEvent>(repairedLineString, _serializerSettings); _processingNotificationsCollector.ReportWarning("Invalid Json found and repaired", _filePath, originalEvent.LineNumber, nameof(NativeJsonLogsReader)); return(new ReadLogLineResult(originalEvent.LineNumber, deserializedObject)); } } catch (JsonException ex) { _processingNotificationsCollector.ReportError(ex.Message, _filePath, originalEvent.LineNumber, nameof(NativeJsonLogsReader)); return(new ReadLogLineResult(originalEvent.LineNumber, null)); } }
public IEnumerable <ReadLogLineResult> ReadLines() { using var reader = new StreamReader(_stream); using var csv = new CsvReader(reader, _csvConfig); var lineNumber = 1; while (csv.Read()) { ReadLogLineResult result = null; try { result = new ReadLogLineResult(lineNumber, csv.GetRecord <T>()); lineNumber++; } catch (Exception ex) when( ex is TypeConverterException || ex is ReaderException) { _processingNotificationsCollector?.ReportError($"Error reading CSV record. {ex.Message}", _filePath, csv.Context.Row, nameof(CsvLogReader <T>)); } if (result != null) { yield return(result); } } }
public TabadminControllerEvent ParseEvent(LogLine logLine, JavaLineMatchResult javaLineMatchResult) { if (string.IsNullOrWhiteSpace(javaLineMatchResult.Message)) { _processingNotificationsCollector.ReportError($"Line does not appear to have a message", logLine, nameof(TabadminControllerEventParser)); return(null); } if (javaLineMatchResult.IsWarningPriorityOrHigher()) { return(new TabadminControllerEvent("Error - Tabadmin Controller", logLine, javaLineMatchResult, _buildTracker)); } if (javaLineMatchResult.Class.Equals("com.tableausoftware.tabadmin.configuration.builder.AppConfigurationBuilder", StringComparison.InvariantCulture)) { return(ParseVersionInfo(logLine, javaLineMatchResult)); } if (javaLineMatchResult.Class.StartsWith("com.tableausoftware.tabadmin.webapp.asyncjobs.", StringComparison.InvariantCulture)) { return(ParseAsyncJobServiceMessages(logLine, javaLineMatchResult)); } if (javaLineMatchResult.Class.StartsWith("com.tableausoftware.tabadmin.webapp.config.")) { return(ParseConfigChangeRequests(logLine, javaLineMatchResult)); } if (javaLineMatchResult.Class.StartsWith("com.tableausoftware.tabadmin.webapp.")) { return(ParseAuthenticationRequests(logLine, javaLineMatchResult)); } return(null); // Line did not match any known events }
private void ParseLinuxLine(LogLine logLine) { var worker = logLine.LogFileInfo.Worker; if (_skipRemainingInput.Contains(worker)) { return; } if (!(logLine.LineContents is string logString)) { _processingNotificationsCollector.ReportError("Received null/non-string netstat data", logLine, nameof(NetstatPlugin)); return; } // starts in Active Internet connections mode, then moves to Active UNIX domain sockets, which we dont care about if (logString.StartsWith("Active UNIX domain sockets")) { _skipRemainingInput.Add(worker); return; } var match = NetstatLinuxInternetConnection.Match(logString); if (match == Match.Empty) { return; } var groups = match.Groups; var processName = groups["program_name"].Value; var parsedResult = new NetstatActiveConnection { FileLastModified = logLine.LogFileInfo.LastModifiedUtc, Line = logLine.LineNumber, Worker = worker, // Process names can be truncated in netstat output IsKnownTableauServerProcess = KnownTableauServerProcesses.Any(p => p.StartsWith(processName)), Protocol = groups["protocol"].Value, RecvQ = int.TryParse(groups["recv_q"].Value, out var rq) ? rq : (int?)null, SendQ = int.TryParse(groups["send_q"].Value, out var sq) ? sq : (int?)null, LocalAddress = groups["local_address"].Value, LocalPort = groups["local_port"].Value, ForeignAddress = groups["foreign_address"].Value, ForeignPort = groups["foreign_port"].Value, TcpState = groups["state"].Value, ProcessId = int.TryParse(groups["pid"].Value, out var pid) ? pid : (int?)null, ProcessName = processName, }; _writer.AddLine(parsedResult); }
public void ProcessLogLine(LogLine logLine, LogType logType) { if (!(logLine.LineContents is NativeJsonLogsBaseEvent baseEvent)) { var errorMessage = $"Was not able to cast line contents as {nameof(NativeJsonLogsBaseEvent)}"; _processingNotificationsCollector.ReportError(errorMessage, logLine, nameof(HyperPlugin)); return; } WriteHyperErrorIfMatch(logLine, baseEvent); WriteHyperQueryIfMatch(logLine, baseEvent); }
public void ProcessLogLine(LogLine logLine, LogType logType) { if (!(logLine.LineContents is PostgresCsvMapping csvMapping)) { _processingNotificationsCollector.ReportError($"Failed to case received line as {nameof(PostgresCsvMapping)}", logLine, nameof(PostgresPlugin)); return; } var duration = (int?)null; var durationMatch = DurationMessageRegex.Match(csvMapping.Message); if (durationMatch.Success && durationMatch.Groups["duration"] != null) { if (int.TryParse(durationMatch.Groups["duration"].Value, out var durationInt)) { duration = durationInt; } } var @event = new PostgresEvent() { Duration = duration, File = logLine.LogFileInfo.FileName, FilePath = logLine.LogFileInfo.FilePath, LineNumber = logLine.LineNumber, Message = csvMapping.Message, ProcessId = csvMapping.Pid, Severity = csvMapping.Sev, Timestamp = csvMapping.Timestamp.ToUniversalTime(), // Postgres logs in GMT and includes timezone, so we need to save it as universal time to preserve original timestamp Worker = logLine.LogFileInfo.Worker, }; _writer.AddLine(@event); }
public void ProcessLogLine(LogLine logLine, LogType logType) { var javaLineMatchResult = logLine.LineContents.MatchJavaLine(TabadminControllerAndAgentJavaRegex); if (!javaLineMatchResult.SuccessfulMatch) { _processingNotificationsCollector.ReportError($"Failed to parse `{logType}` event from log line", logLine, nameof(TabadminControllerPlugin)); return; } var parsedEvent = logType switch { LogType.ControlLogsJava => ParseError(logLine, javaLineMatchResult, "Error - Control Logs"), LogType.TabadminAgentJava => ParseError(logLine, javaLineMatchResult, "Error - Tabadmin Agent"), LogType.TabadminControllerJava => _tabadminControllerEventParser.ParseEvent(logLine, javaLineMatchResult), _ => throw new LogSharkProgramLogicException( $"{nameof(TabadminControllerPlugin)} received log line of `{logType}` type, but is not configured to process it") }; if (parsedEvent != null) { _writer.AddLine(parsedEvent); } }
public void ProcessLogLine(LogLine logLine, LogType logType) { var match = logLine.LineContents.CastToStringAndRegexMatch(SharedRegex.JavaLogLineRegex); if (match == null || !match.Success) { _processingNotificationsCollector.ReportError("Failed to parse Vizportal Java event from log line", logLine, nameof(VizportalPlugin)); return; } var @event = new VizportalEvent() { Class = match.GetString("class"), File = logLine.LogFileInfo.FileName, FilePath = logLine.LogFileInfo.FilePath, LineNumber = logLine.LineNumber, Message = match.GetString("message"), RequestId = match.GetString("req"), SessionId = match.GetString("sess"), Severity = match.GetString("sev"), Site = match.GetNullableString("site"), Timestamp = TimestampParsers.ParseJavaLogsTimestamp(match.GetString("ts")), User = match.GetString("user"), Worker = logLine.LogFileInfo.Worker, }; _writer.AddLine(@event); }
public void ProcessLogLine(LogLine logLine, LogType logType) { if (!(logLine.LineContents is NativeJsonLogsBaseEvent baseEvent)) { var errorMessage = $"Was not able to cast line contents as {nameof(NativeJsonLogsBaseEvent)}"; _processingNotificationsCollector.ReportError(errorMessage, logLine, nameof(ResourceManagerPlugin)); return; } if (!CanContainResourceManagerInfo(baseEvent, logType)) { return; } var message = GetSrmMessage(baseEvent, logType, logLine); var processName = ProcessInfoParser.GetProcessName(logType); _eventsProcessor.ProcessEvent(baseEvent, message, logLine, processName); }
// This plugin is a little special, as it does part of its work at dispose. // It writes all config entries as it encounters them, but also keeps cache of all processed config files inside Dictionary. This works fine because config files are fairly small. // Then on Dispose it analyzes collected files to generate and write Process Topology information public void ProcessLogLine(LogLine logLine, LogType logType) { if (logLine.LineContents is Dictionary <string, string> configContents) { var configFile = new ConfigFile(logLine.LogFileInfo, configContents, logType); var configEntriesWritten = WriteConfigFileEntries(configFile); if (configEntriesWritten > 0) { _processedConfigFiles.Add(configFile); _logger.LogInformation("{configEntriesWritten} config entries processed and written", configEntriesWritten); } } else { const string error = "Failed to interpret input as Dictionary<string, string>. Either log set contains empty/corrupt yaml config files or incorrect log reader is used for the plugin"; _processingNotificationsCollector.ReportError(error, logLine, nameof(ConfigPlugin)); } }
public void ProcessLogLine(LogLine logLine, LogType logType) { if (!(logLine.LineContents is string logLineAsString)) { _processingNotificationsCollector.ReportError("Failed to parse log line as string", logLine, nameof(TabadminPlugin)); return; } var logLineMatch = logLineAsString.GetRegexMatchAndMoveCorrectRegexUpFront(_logLineRegexes); if (logLineMatch == null || !logLineMatch.Success) { _processingNotificationsCollector.ReportError("Failed to parse log line as Tabadmin event", logLine, nameof(TabadminPlugin)); return; } WriteTableauServerVersionEventIfMatch(logLine, logLineMatch); WriteTabadminActionEventIfMatch(logLine, logLineMatch); WriteTabadminErrorEventIfMatch(logLine, logLineMatch); }
public void ProcessLogLine(LogLine logLine, LogType logType) { if (!(logLine.LineContents is NativeJsonLogsBaseEvent baseEvent)) { var errorMessage = $"Was not able to cast line contents as {nameof(NativeJsonLogsBaseEvent)}"; _processingNotificationsCollector.ReportError(errorMessage, logLine, nameof(VizqlDesktopPlugin)); return; } _eventProcessor.ProcessEvent(baseEvent, logLine); }
public void ProcessLogLine(LogLine logLine, LogType logType) { var logLineText = logLine.LineContents as string; if (logLineText == null) { _processingNotificationsCollector.ReportError("Failed to parse log line as string", logLine, nameof(BackgrounderPlugin)); return; } _backgrounderEventParser.ParseAndPersistLine(logLine, logLineText); }
public void ProcessLogLine(LogLine logLine, LogType logType) { var @event = ParseEvent(logLine); if (@event == null) { _processingNotificationsCollector.ReportError("Failed to parse Filestore event from log line", logLine, nameof(FilestorePlugin)); return; } _writer.AddLine(@event); }
private void ProcessCppLine(LogLine logLine) { if (!(logLine.LineContents is NativeJsonLogsBaseEvent baseEvent)) { _processingNotificationsCollector.ReportError($"Was not able to cast line contents as {nameof(NativeJsonLogsBaseEvent)}", logLine, nameof(DataServerPlugin)); return; } if (baseEvent.EventPayload == null) { _processingNotificationsCollector.ReportError("Log line does not contain event payload (\"v\" key in json). Skipping this line", logLine, nameof(DataServerPlugin)); return; } if (ShouldSkip(baseEvent)) { return; } var @event = new DataServerEvent(logLine, baseEvent); _writer.AddLine(@event); }
public void ProcessLogLine(LogLine logLine, LogType logType) { var javaLineMatchResult = logLine.LineContents.MatchJavaLine(_regex); if (!javaLineMatchResult.SuccessfulMatch) { _processingNotificationsCollector.ReportError($"Failed to parse Filestore event from log line", logLine, nameof(FilestorePlugin)); return; } var @event = new FilestoreEvent(logLine, javaLineMatchResult); _writer.AddLine(@event); }
public void ProcessLogLine(LogLine logLine, LogType logType) { var javaLineMatchResult = logLine.LineContents.MatchJavaLineWithSessionInfo(SharedRegex.JavaLogLineRegex); if (!javaLineMatchResult.SuccessfulMatch) { _processingNotificationsCollector.ReportError("Failed to parse Vizportal Java event from log line", logLine, nameof(VizportalPlugin)); return; } var @event = new VizportalEvent(logLine, javaLineMatchResult); _writer.AddLine(@event); }
private void AddBuildRecordSafely(long roundedTimestamp, string build, DateTime originalTimestamp) { if (_timestampsForBuilds.ContainsKey(roundedTimestamp)) { if (_timestampsForBuilds[roundedTimestamp] != build) { _processingNotificationsCollector.ReportError($"Timestamp `{originalTimestamp}` contains reference for build `{build}`, however another build - `{_timestampsForBuilds[roundedTimestamp]}` - is registered within two minutes of this event. Build output around this time could be inconsistent.", nameof(BuildTracker)); } return; } _timestampsForBuilds.Add(roundedTimestamp, build); }
public void ProcessLogLine(LogLine logLine, LogType logType) { if (!(logLine.LineContents is NativeJsonLogsBaseEvent baseEvent)) { var errorMessage = $"Was not able to cast line contents as {nameof(NativeJsonLogsBaseEvent)}"; _processingNotificationsCollector.ReportError(errorMessage, logLine, nameof(ArtPlugin)); return; } if (baseEvent.ArtData == null) { return; // This is normal - only small subset of log lines has ART data } ArtData artData; try { artData = baseEvent.ArtData.ToObject <ArtData>(_jsonSerializer); } catch (JsonException ex) { _processingNotificationsCollector.ReportError(ex.Message, logLine, nameof(ArtPlugin)); return; } var @event = new FlattenedArtEvent(artData, baseEvent, logLine); if (logType == LogType.VizqlDesktop) { _desktopWriter.AddLine(@event); } else { _serverWriter.AddLine(@event); } }
public void ProcessLogLine(LogLine logLine, LogType logType) { var @event = ApacheEventParser.ParseEvent(logLine); if (@event == null) { _processingNotificationsCollector.ReportError("Failed to parse Apache event from log line", logLine, nameof(ApachePlugin)); return; } if (_includeGatewayHealthChecks || !IsHealthCheckRequest(@event)) { _writer.AddLine(@event); } }
public IEnumerable <ReadLogLineResult> ReadLines() { using (var reader = new StreamReader(_stream)) { string line = null; var lineCount = 0; while (!_passedHeader && !reader.EndOfStream) { line = reader.ReadLine(); lineCount++; if (HeaderLineRegex.IsMatch(line)) { _passedHeader = true; } } if (!_passedHeader) { _processingNotificationsCollector.ReportError("Failed to find header before reaching EOF. Netstat info will be empty", _filePath, 0, nameof(NetstatWindowsReader)); } var section = new Stack <(string line, int lineNumber)>(); var sectionStartLine = lineCount + 1; while (line != null && !reader.EndOfStream) { line = reader.ReadLine(); lineCount++; if (!string.IsNullOrWhiteSpace(line)) { section.Push((line, lineCount)); if (ExecutableInfoRegex.IsMatch(line)) { yield return(new ReadLogLineResult(sectionStartLine, section)); section = new Stack <(string line, int lineNumber)>(); sectionStartLine = lineCount + 1; } } } } }
private void AddCpuSampleEvent(NativeJsonLogsBaseEvent baseEvent, string message, LogLine logLine, string processName) { ResourceManagerCpuSample record = null; var currentAndTotalMatch = CurrentAndTotalCpuUtilRegex.Match(message); if (currentAndTotalMatch.Success) { record = ResourceManagerCpuSample.GetEventWithNullCheck( baseEvent, logLine, processName, TryParseIntWithLogging(currentAndTotalMatch, "current_process_util", logLine), TryParseIntWithLogging(currentAndTotalMatch, "total_processes_util", logLine) ); _cpuSamplesWriter.AddLine(record); return; } var currentMatch = CurrentCpuUtilRegex.Match(message); if (currentMatch.Success) { record = ResourceManagerCpuSample.GetEventWithNullCheck( baseEvent, logLine, processName, TryParseIntWithLogging(currentMatch, "current_process_util", logLine), null ); _cpuSamplesWriter.AddLine(record); return; } _processingNotificationsCollector.ReportError("Failed to process line as CpuSampleEvent.", logLine, nameof(ResourceManagerPlugin)); }
private IEnumerable <LogSetInfo> HandleNestedZips(string zippedLogSetPath, string tempDirRoot, IProcessingNotificationsCollector processingNotificationsCollector) { _logger.LogInformation("Looking for known nested zip files inside zip file '{logSetPath}'", zippedLogSetPath); using var zip = ZipFile.Open(zippedLogSetPath, ZipArchiveMode.Read); var nestedZips = zip.Entries.Where(IsNestedZip).ToList(); var nestedZipFilesInfo = new List <LogSetInfo>(); if (nestedZips.Count == 0) { _logger.LogInformation("No known nested zip files found in '{logSetPath}", zippedLogSetPath); return(nestedZipFilesInfo); } var tempDir = Path.Combine(tempDirRoot, NestedZipTempDirName); Directory.CreateDirectory(tempDir); _tempFileTracker.AddDirectory(tempDir); foreach (var nestedZip in nestedZips) { var extractPath = Path.Combine(tempDir, nestedZip.Name); _logger.LogInformation("Extracting nested archive {nestedZipName} into '{nestedZipExtractPath}'", nestedZip.Name, extractPath); nestedZip.ExtractToFile(extractPath); _logger.LogInformation("Successfully extracted {nestedZipName} into '{nestedZipExtractPath}'", nestedZip.Name, extractPath); var checkResult = FileCanBeOpened(extractPath, _logger); if (checkResult.FileCanBeOpened) { nestedZipFilesInfo.Add(new LogSetInfo(extractPath.NormalizeSeparatorsToUnix(), nestedZip.FullName.RemoveZipFromTail(), true, _rootPath)); } else { var error = $"Nested zip \"{nestedZip.FullName}\" appears to be corrupt. It will not be processed. Underlying error: {checkResult.ErrorMessage ?? "(null)"}"; processingNotificationsCollector.ReportError(error, nestedZip.FullName, 0, nameof(TableauLogsExtractor)); } } _logger.LogInformation("Found and extracted {numberOfNestedZips} known nested zip files inside `{logSetPath}`", nestedZipFilesInfo.Count, zippedLogSetPath); return(nestedZipFilesInfo); }
private IDictionary <int, string> GetWorkerHostnameMap(ConfigFile configFile, IProcessingNotificationsCollector processingNotificationsCollector) { if (configFile == null || !configFile.Values.ContainsKey(ConfigKeyWithHostMapping)) { var error = $"Config file is null or doesn't contain {ConfigKeyWithHostMapping} key. Cannot generate hostname map for workers"; processingNotificationsCollector.ReportError(error, nameof(ConfigPlugin)); return(new Dictionary <int, string>()); } var workerHostsLine = configFile.Values[ConfigKeyWithHostMapping]; return(workerHostsLine .Split(',') .Select((hostName, ind) => new { Index = ind, HostName = hostName.Trim() }) .ToDictionary(obj => obj.Index, obj => obj.HostName)); }
public IList <ConfigProcessInfo> GenerateProcessInfoRecords() { if (_workgroupYml == null || _hostnameMap.Count == 0) { const string error = "Config file is null or hostname map is empty. Cannot generate process info records"; _processingNotificationsCollector.ReportError(error, nameof(ConfigPlugin)); return(new List <ConfigProcessInfo>()); } var processInfoRecords = new List <ConfigProcessInfo>(); for (var workerIndex = 0; workerIndex < _hostnameMap.Count; workerIndex++) { processInfoRecords.AddRange(GetWorkerDetails(workerIndex)); } // Add process info records that work differently than other processes processInfoRecords.AddRange(GetPostgresProcessDetails()); return(processInfoRecords); }
private void ProcessClusterControllerLine(LogLine logLine) { var javaLineMatchResult = logLine.LineContents.MatchJavaLine(_clusterControllerLogsRegex); if (!javaLineMatchResult.SuccessfulMatch) { _processingNotificationsCollector.ReportError("Failed to process line as Cluster Controller event", logLine, nameof(ClusterControllerPlugin)); return; } // 2018.2 linux/node1/clustercontroller_0.20182.18.0627.22308809190037074300891/logs/clustercontroller.log:333:2018-08-08 11:10:12.705 +1000 pool-18-thread-1 ERROR : com.tableausoftware.cluster.http.HttpServiceMonitor - IOException connecting to HTTP server at http://localhost:8000/favicon.ico if (javaLineMatchResult.IsErrorPriorityOrHigher()) { var entry = new ClusterControllerError(logLine, javaLineMatchResult); _errorWriter.AddLine(entry); } // 2018.2 linux/node2/clustercontroller_0.20182.18.0627.22301467407848617992908/logs/clustercontroller.log:2018-08-08 15:04:51.901 +1000 Thread-6 INFO: com.tableausoftware.cluster.postgres.PostgresManager - PostgresManager stop if (javaLineMatchResult.Class == "com.tableausoftware.cluster.postgres.PostgresManager" && _postgresMessages.ContainsKey(javaLineMatchResult.Message)) { var entry = new ClusterControllerPostgresAction(logLine, javaLineMatchResult.Timestamp) { Action = _postgresMessages.TryGetValue(javaLineMatchResult.Message, out var pm) ? pm : null, }; _postgresActionWriter.AddLine(entry); } if (javaLineMatchResult.Class == "com.tableausoftware.cluster.storage.DiskSpaceMonitor" && javaLineMatchResult.Message.StartsWith(DiskSpaceMonitorPrefix, StringComparison.Ordinal)) { var diskSpaceStats = _diskSpaceMessageRegex.Match(javaLineMatchResult.Message); if (!diskSpaceStats.Success) { _processingNotificationsCollector.ReportError("Failed to parse Cluster Controller DiskSpaceMonitor event from log line", logLine, nameof(ClusterControllerPlugin)); return; } long?usedSpace = diskSpaceStats.GetNullableLong("usedSpace"); ClusterControllerDiskSpaceSample entry; // this if check is a workaround for the fact that DiskSpaceMonitor.java has a bug on this line // due to it missing the disk parameter: // m_logger.info("disk {}: total space={} used space={}", totalSpace, usedSpace); if (usedSpace != null) { entry = new ClusterControllerDiskSpaceSample(logLine, javaLineMatchResult.Timestamp) { Disk = diskSpaceStats.GetString("disk"), TotalSpace = diskSpaceStats.GetNullableLong("totalSpace"), UsedSpace = diskSpaceStats.GetNullableLong("usedSpace"), }; } else { entry = new ClusterControllerDiskSpaceSample(logLine, javaLineMatchResult.Timestamp) { Disk = null, TotalSpace = diskSpaceStats.GetNullableLong("disk"), UsedSpace = diskSpaceStats.GetNullableLong("totalSpace"), }; } _diskSpaceSampleWriter.AddLine(entry); } // 2018.2 linux/node2/clustercontroller_0.20182.18.0627.22301467407848617992908/logs/clustercontroller.log:262:2018-08-08 12:11:23.531 +1000 pool-25-thread-1 INFO : com.tableausoftware.cluster.storage.DiskIOMonitor - disk I/O 1min avg > device:/dev/vda1, reads:0.00, readBytes:0.00, writes:0.35, writeBytes:7645.87, queue:0.00 // 2018.2_windows/node1/clustercontroller_0.20182.18.1001.2115746959230678183412/logs/clustercontroller.log:5:2018-10-03 00:02:46.613 +0000 pool-24-thread-1 INFO : com.tableausoftware.cluster.storage.DiskIOMonitor - disk I/O 1min avg > device:C:\, reads:0.00, readBytes:0.00, writes:0.00, writeBytes:0.00, queue:0.00 if (javaLineMatchResult.Severity == "INFO" && javaLineMatchResult.Class == "com.tableausoftware.cluster.storage.DiskIOMonitor" && javaLineMatchResult.Message.StartsWith(DiskIoMonitorMessagePrefix, StringComparison.Ordinal)) { var diskStats = _diskIoMessageRegex.Match(javaLineMatchResult.Message); if (!diskStats.Success) { _processingNotificationsCollector.ReportError("Failed to parse Cluster Controller DiskIOMonitor event from log line", logLine, nameof(ClusterControllerPlugin)); return; } // Numbers logged in this even are locale-specific, so we need to use method with normalization on them. Only ClusterController logs appear to log locale-sensitive numbers var entry = new ClusterControllerDiskIoSample(logLine, javaLineMatchResult.Timestamp) { Device = diskStats.GetString("device"), ReadsPerSec = diskStats.GetNullableDoubleWithDelimiterNormalization("reads"), ReadBytesPerSec = diskStats.GetNullableDoubleWithDelimiterNormalization("readBytes"), WritesPerSec = diskStats.GetNullableDoubleWithDelimiterNormalization("writes"), WriteBytesPerSec = diskStats.GetNullableDoubleWithDelimiterNormalization("writeBytes"), QueueLength = diskStats.GetNullableDoubleWithDelimiterNormalization("queue"), }; _diskIOSampleWriter.AddLine(entry); } }
public void ProcessLogLine(LogLine logLine, LogType logType) { if (!(logLine.LineContents is NativeJsonLogsBaseEvent jsonEvent)) { var errorMessage = $"Was not able to cast line contents as {nameof(NativeJsonLogsBaseEvent)}"; _processingNotificationsCollector.ReportError(errorMessage, logLine, nameof(ServerTelemetryPlugin)); return; } if (jsonEvent.EventType != "server-telemetry") { return; } if (jsonEvent.EventPayload == null) { _processingNotificationsCollector.ReportError("ServerTelemetry event has no payload", logLine, nameof(ServerTelemetryPlugin)); return; } var jsonMessage = jsonEvent.EventPayload.ToObject <ServerTelemetryEventMessageJson>(); var jsonMetrics = ParseMetrics(jsonMessage, jsonMessage.RequestInfo.RequestId); _metricWriter.AddLines(jsonMetrics); var processString = ProcessRegex.Match(jsonMessage.SessionId).Groups["process"].Value; var process = int.TryParse(processString, out int p) ? p : (int?)null; var @event = new ServerTelemetryEvent(logLine, jsonEvent.Timestamp) { ActionName = jsonMessage.RequestInfo.ActionName, ActionSizeBytes = jsonMessage.RequestInfo.ActionSizeBytes, ActionType = jsonMessage.RequestInfo.ActionType, AnnotationCount = jsonMessage.RequestInfo.AnnotationCount, ClientRenderMode = jsonMessage.RequestInfo.ClientRenderMode, CustomShapeCount = jsonMessage.RequestInfo.CustomShapeCount, CustomShapePixelCount = jsonMessage.RequestInfo.CustomShapePixelCount, DevicePixelRatio = jsonMessage.DevicePixelRatio, DsdDeviceType = jsonMessage.DsdDeviceType, EncodingCount = jsonMessage.RequestInfo.EncodingCount, FilterFieldCount = jsonMessage.RequestInfo.FilterFieldCount, Height = jsonMessage.RequestInfo.Height, IsDashboard = jsonMessage.RequestInfo.IsDashboard, MarkCount = jsonMessage.RequestInfo.MarkCount, MarkLabelCount = jsonMessage.RequestInfo.MarkLabelCount, NodeCount = jsonMessage.RequestInfo.NodeCount, NumViews = jsonMessage.RequestInfo.NumViews, NumZones = jsonMessage.RequestInfo.NumZones, PaneCount = jsonMessage.RequestInfo.PaneCount, Process = process, ProcessId = jsonEvent.ProcessId, ReflineCount = jsonMessage.RequestInfo.ReflineCount, RepositoryURL = jsonMessage.RequestInfo.RepositoryURL, RequestId = jsonMessage.RequestInfo.RequestId, SessionId = jsonEvent.SessionId, SessionIdInMessage = jsonMessage.SessionId, SessionState = jsonMessage.RequestInfo.SessionState, SheetName = jsonMessage.RequestInfo.SheetName, SiteName = jsonMessage.SiteName, TextMarkCount = jsonMessage.RequestInfo.TextMarkCount, ThreadId = jsonEvent.ThreadId, TooltipCount = jsonMessage.RequestInfo.TooltipCount, TransparentLinemarkCount = jsonMessage.RequestInfo.TransparentLinemarkCount, UserAgent = jsonMessage.UserAgent, UserName = jsonMessage.UserName, VertexCount = jsonMessage.RequestInfo.VertexCount, Width = jsonMessage.RequestInfo.Width, WorkbookName = jsonMessage.WorkbookName, }; _eventWriter.AddLine(@event); }
private void ProcessClusterControllerLine(LogLine logLine) { var match = logLine.LineContents.CastToStringAndRegexMatch(_clusterControllerLogsRegex); if (match == null || match == Match.Empty) { _processingNotificationsCollector.ReportError("Failed to process line as Cluster Controller event", logLine, nameof(ClusterControllerPlugin)); return; } var groups = match.Groups; var ts = groups["ts"].Value; var sev = groups["sev"].Value; var @class = groups["class"].Value; var message = groups["message"].Value; // 2018.2 linux/node1/clustercontroller_0.20182.18.0627.22308809190037074300891/logs/clustercontroller.log:333:2018-08-08 11:10:12.705 +1000 pool-18-thread-1 ERROR : com.tableausoftware.cluster.http.HttpServiceMonitor - IOException connecting to HTTP server at http://localhost:8000/favicon.ico if (sev == "ERROR" || sev == "FATAL") { var entry = new ClusterControllerError(logLine, ts) { Severity = sev, Message = message, Class = @class, }; _errorWriter.AddLine(entry); } // 2018.2 linux/node2/clustercontroller_0.20182.18.0627.22301467407848617992908/logs/clustercontroller.log:2018-08-08 15:04:51.901 +1000 Thread-6 INFO: com.tableausoftware.cluster.postgres.PostgresManager - PostgresManager stop if (@class == "com.tableausoftware.cluster.postgres.PostgresManager" && _postgresMessages.ContainsKey(message)) { var entry = new ClusterControllerPostgresAction(logLine, ts) { Action = _postgresMessages.TryGetValue(message, out var pm) ? pm : null, }; _postgresActionWriter.AddLine(entry); } // 2018.2 linux/node2/clustercontroller_0.20182.18.0627.22301467407848617992908/logs/clustercontroller.log:262:2018-08-08 12:11:23.531 +1000 pool-25-thread-1 INFO : com.tableausoftware.cluster.storage.DiskIOMonitor - disk I/O 1min avg > device:/dev/vda1, reads:0.00, readBytes:0.00, writes:0.35, writeBytes:7645.87, queue:0.00 // 2018.2_windows/node1/clustercontroller_0.20182.18.1001.2115746959230678183412/logs/clustercontroller.log:5:2018-10-03 00:02:46.613 +0000 pool-24-thread-1 INFO : com.tableausoftware.cluster.storage.DiskIOMonitor - disk I/O 1min avg > device:C:\, reads:0.00, readBytes:0.00, writes:0.00, writeBytes:0.00, queue:0.00 if (sev == "INFO" && @class == "com.tableausoftware.cluster.storage.DiskIOMonitor" && message.StartsWith(DiskIoMonitorMessagePrefix, StringComparison.Ordinal)) { var diskStats = _diskIoMessageRegex.Match(message); if (!diskStats.Success) { _processingNotificationsCollector.ReportError("Failed to parse Cluster Controller DiskIOMonitor event from log line", logLine, nameof(ClusterControllerPlugin)); return; } // Numbers logged in this even are locale-specific, so we need to use method with normalization on them. Only ClusterController logs appear to log locale-sensitive numbers var entry = new ClusterControllerDiskIoSample(logLine, ts) { Device = diskStats.GetString("device"), ReadsPerSec = diskStats.GetNullableDoubleWithDelimiterNormalization("reads"), ReadBytesPerSec = diskStats.GetNullableDoubleWithDelimiterNormalization("readBytes"), WritesPerSec = diskStats.GetNullableDoubleWithDelimiterNormalization("writes"), WriteBytesPerSec = diskStats.GetNullableDoubleWithDelimiterNormalization("writeBytes"), QueueLength = diskStats.GetNullableDoubleWithDelimiterNormalization("queue"), }; _diskIOSampleWriter.AddLine(entry); } }