private static Task InternalParse(XMLLogParser parser, TransientParserConfigs config, Stream logStream, CancellationToken cancellationToken) { var parserStream = !config.LogHasRoot ? Encoding.UTF8.GetBytes("<root>").Append(logStream) : logStream; if (!config.LogHasRoot) { parserStream = parserStream.Append(Encoding.UTF8.GetBytes("</root>")); } var parseTask = Task.Run(() => { try { ParseLoop(parser, config, parserStream, cancellationToken); } catch { //XXX probably want to do something here... } }, cancellationToken); if (!config.LogHasRoot) { // If we didn't have the root, then we end up creating a Stream by appending streams together and need to dispose it return(parseTask.ContinueWith(_ => parserStream.Dispose(), TaskContinuationOptions.ExecuteSynchronously)); } return(parseTask); }
private static LogParserErrors ProcessValidElement(XMLLogParser parser, TransientParserConfigs config, XElement element) { if (!parser.validConfigs) { return(LogParserErrors.ConfigNotInitialized); } if (parser.registry == null) { return(LogParserErrors.RegistryNotSet); } if (parser.timestampPath == null || parser.messagePath == null) { return(LogParserErrors.ConfigValueInvalid); } var timestamp = GetElementDataFromPath(parser.timestampPath, element); if (string.IsNullOrWhiteSpace(timestamp)) { return(LogParserErrors.MissingTimestamp); } var message = GetElementDataFromPath(parser.messagePath, element); if (message == null) { return(LogParserErrors.MissingMessage); } var entry = parser.registry.AddLog(timestamp, message); return(ProcessCommonLogAttributes(parser, config, entry, element)); }
/// <summary> /// Set the configurations to apply to the log parser. /// </summary> /// <param name="config">Configurations to apply to the log parser.</param> public void SetConfig(LogConfig config) { if (validConfigs && attributePaths != null) { throw new InvalidOperationException("Can only set config once"); } validConfigs = config.IsValid; timestampPath = ParserUtil.ParsePath(config.TimestampPath); messagePath = ParserUtil.ParsePath(config.LogMessagePath); setConfigFailureHandlingCounter.Increment(config.ParsingFailureHandling.ToString()); defaultTransientConfig = new TransientParserConfigs() { FailureHandling = config.ParsingFailureHandling, LogHasRoot = config.LogHasRoot }; attributePaths = new Dictionary <LogAttribute, ParserPathElement[]>(); #if NETSTANDARD2_0 var logConfigType = typeof(LogConfig); foreach (var attribute in LogConfig.GetAttributePaths()) { if (attribute.Key == LogAttribute.Timestamp || attribute.Key == LogAttribute.Message) { continue; } AddAttributePath(attribute.Key, logConfigType.GetProperty(attribute.Value).GetValue(config) as string); } #else AddAttributePath(LogAttribute.ThreadID, config.ThreadIDPath); AddAttributePath(LogAttribute.SourceFile, config.SourceFilePath); AddAttributePath(LogAttribute.Function, config.FunctionPath); AddAttributePath(LogAttribute.SourceLine, config.LogLinePath); AddAttributePath(LogAttribute.Level, config.LogLevelPath); AddAttributePath(LogAttribute.SequenceNumber, config.LogSequencePath); AddAttributePath(LogAttribute.Module, config.ModulePath); AddAttributePath(LogAttribute.Type, config.LogTypePath); AddAttributePath(LogAttribute.Section, config.SectionPath); AddAttributePath(LogAttribute.TraceID, config.TraceIdPath); AddAttributePath(LogAttribute.Context, config.ContextPath); #endif }
private static LogParserErrors ProcessElement(XMLLogParser parser, TransientParserConfigs config, XElement element) { processElementCounter.Increment(); var result = ProcessValidElement(parser, config, element); if (result != LogParserErrors.OK && !ParserUtil.IsFatal(result)) { if (config.FailureHandling == LogParserFailureHandling.SkipEntries) { return(LogParserErrors.OK); } else if (config.FailureHandling == LogParserFailureHandling.MarkEntriesAsFailed) { return(ProcessInvalidElement(parser, config, element)); } } return(result); }
private static void InternalApplyContextConfig(XMLLogParser parser, TransientParserConfigs priorConfig, IDictionary <ContextConfigs, object> config, Action <ILogParser> context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } var contextParser = new XMLContextParser(parser, priorConfig); contextParser.ApplyConfig(config); try { context(contextParser); } finally { contextParser.IsUsable = false; } }
private static LogParserErrors ProcessInvalidElement(XMLLogParser parser, TransientParserConfigs config, XElement element) { var entry = parser.registry.AddFailedLog(); var timestamp = GetElementDataFromPath(parser.timestampPath, element); if (!string.IsNullOrWhiteSpace(timestamp) && DateTime.TryParse(timestamp, out DateTime time)) { parser.registry.AddValueToLog(entry, LogAttribute.Timestamp, time); } var message = GetElementDataFromPath(parser.messagePath, element); if (message != null) { parser.registry.AddValueToLog(entry, LogAttribute.Message, message); } var result = ProcessCommonLogAttributes(parser, config, entry, element); parser.registry.NotifyFailedLogParsed(entry); return(result); }
public XMLContextParser(XMLLogParser parser, TransientParserConfigs priorConfig) { this.parser = parser; this.config = priorConfig; }
private static void ParseLoop(XMLLogParser parser, TransientParserConfigs config, Stream input, CancellationToken cancellationToken) { using (var xmlReader = XmlReader.Create(input)) { // Don't just use XML's natual tree-creation setup... because we don't want the root element to have children. For large or streaming logs, it will become a resource hog var elements = new Stack <XElement>(); bool exitLoop = false; while (!exitLoop && !cancellationToken.IsCancellationRequested && xmlReader.Read()) { switch (xmlReader.NodeType) { case XmlNodeType.Element: var name = XName.Get(xmlReader.Name, xmlReader.NamespaceURI); var element = new XElement(name); if (elements.Count > 1) // Don't add children to root to ensure it doesn't become massive { elements.Peek().Add(element); } elements.Push(element); if (xmlReader.HasAttributes) { while (xmlReader.MoveToNextAttribute()) { var attName = XName.Get(xmlReader.Name, xmlReader.NamespaceURI); var att = new XAttribute(attName, xmlReader.Value); elements.Peek().Add(att); } xmlReader.MoveToElement(); } break; case XmlNodeType.EndElement: xmlElementCounter.Increment(); if (xmlReader.Name == elements.Peek().Name) { var finishedElement = elements.Pop(); if (elements.Count == 1) // Don't process the elements unless they're children of root. Anything else is a child element of a log element { using (processElementTimer.NewContext()) { if (ProcessElement(parser, config, finishedElement) != LogParserErrors.OK) { exitLoop = true; break; } } } } else { Console.Error.WriteLine($"Element {elements.Peek().Name} ended, but the name of the ending element {xmlReader.Name} doesn't match. Possibly out of sync..."); } break; case XmlNodeType.CDATA: xmlCDATACounter.Increment(); elements.Peek().Add(new XCData(xmlReader.Value)); break; case XmlNodeType.Text: xmlTextCounter.Increment(); elements.Peek().Add(new XText(xmlReader.Value)); break; case XmlNodeType.Whitespace: break; default: unknownXmlElementCounter.Increment(xmlReader.NodeType.ToString()); break; } } // Only do if cancellation wasn't requested because it means we ran to completion but never found root. Cancellation means we never will. if (elements.Count != 0 && !cancellationToken.IsCancellationRequested) { xmlRootUnfinishedCounter.Increment(); Console.Error.WriteLine("Root element didn't end"); } } }
private static LogParserErrors ProcessCommonLogAttributes(XMLLogParser parser, TransientParserConfigs config, ILogEntry entry, XElement element) { var attributesAdded = 0; foreach (var kv in parser.attributePaths) { var rawValue = GetElementDataFromPath(kv.Value, element); var fieldType = ParserPathElementFieldType.String; if (kv.Value.Length != 0) { fieldType = kv.Value.Last().FieldType; } var value = ParserUtil.CastField(rawValue, fieldType); if (value != null) { //XXX should probably have some test for checking if the value couldn't be added if (parser.registry.AddValueToLog(entry, kv.Key, value)) { attributesAdded++; } else { failureToAddAttributeToLogCounter.Increment(kv.Key.ToString()); } } } if (config.AdditionalAttributes != null) { foreach (var kv in config.AdditionalAttributes) { if (parser.registry.AddValueToLog(entry, kv.Key, kv.Value)) { attributesAdded++; } else { failureToAddAttributeToLogCounter.Increment(kv.Key.ToString()); } } } attributesParsedHistogram.Update(attributesAdded); return(LogParserErrors.OK); }