private static Task InternalParse(XMLLogParser parser, TransientParserConfigs config, Stream logStream, CancellationToken cancellationToken) { var parserStream = !config.LogHasRoot ? Encoding.UTF8.GetBytes("<root>").Append(logStream) : logStream; if (!config.LogHasRoot) { parserStream = parserStream.Append(Encoding.UTF8.GetBytes("</root>")); } var parseTask = Task.Run(() => { try { ParseLoop(parser, config, parserStream, cancellationToken); } catch { //XXX probably want to do something here... } }, cancellationToken); if (!config.LogHasRoot) { // If we didn't have the root, then we end up creating a Stream by appending streams together and need to dispose it return(parseTask.ContinueWith(_ => parserStream.Dispose(), TaskContinuationOptions.ExecuteSynchronously)); } return(parseTask); }
private static LogParserErrors ProcessValidElement(XMLLogParser parser, TransientParserConfigs config, XElement element) { if (!parser.validConfigs) { return(LogParserErrors.ConfigNotInitialized); } if (parser.registry == null) { return(LogParserErrors.RegistryNotSet); } if (parser.timestampPath == null || parser.messagePath == null) { return(LogParserErrors.ConfigValueInvalid); } var timestamp = GetElementDataFromPath(parser.timestampPath, element); if (string.IsNullOrWhiteSpace(timestamp)) { return(LogParserErrors.MissingTimestamp); } var message = GetElementDataFromPath(parser.messagePath, element); if (message == null) { return(LogParserErrors.MissingMessage); } var entry = parser.registry.AddLog(timestamp, message); return(ProcessCommonLogAttributes(parser, config, entry, element)); }
private static LogParserErrors ProcessElement(XMLLogParser parser, TransientParserConfigs config, XElement element) { processElementCounter.Increment(); var result = ProcessValidElement(parser, config, element); if (result != LogParserErrors.OK && !ParserUtil.IsFatal(result)) { if (config.FailureHandling == LogParserFailureHandling.SkipEntries) { return(LogParserErrors.OK); } else if (config.FailureHandling == LogParserFailureHandling.MarkEntriesAsFailed) { return(ProcessInvalidElement(parser, config, element)); } } return(result); }
private static LogParserErrors ProcessCommonLogAttributes(XMLLogParser parser, TransientParserConfigs config, ILogEntry entry, XElement element) { var attributesAdded = 0; foreach (var kv in parser.attributePaths) { var rawValue = GetElementDataFromPath(kv.Value, element); var fieldType = ParserPathElementFieldType.String; if (kv.Value.Length != 0) { fieldType = kv.Value.Last().FieldType; } var value = ParserUtil.CastField(rawValue, fieldType); if (value != null) { //XXX should probably have some test for checking if the value couldn't be added if (parser.registry.AddValueToLog(entry, kv.Key, value)) { attributesAdded++; } else { failureToAddAttributeToLogCounter.Increment(kv.Key.ToString()); } } } if (config.AdditionalAttributes != null) { foreach (var kv in config.AdditionalAttributes) { if (parser.registry.AddValueToLog(entry, kv.Key, kv.Value)) { attributesAdded++; } else { failureToAddAttributeToLogCounter.Increment(kv.Key.ToString()); } } } attributesParsedHistogram.Update(attributesAdded); return(LogParserErrors.OK); }
private static void InternalApplyContextConfig(XMLLogParser parser, TransientParserConfigs priorConfig, IDictionary <ContextConfigs, object> config, Action <ILogParser> context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } var contextParser = new XMLContextParser(parser, priorConfig); contextParser.ApplyConfig(config); try { context(contextParser); } finally { contextParser.IsUsable = false; } }
private static LogParserErrors ProcessInvalidElement(XMLLogParser parser, TransientParserConfigs config, XElement element) { var entry = parser.registry.AddFailedLog(); var timestamp = GetElementDataFromPath(parser.timestampPath, element); if (!string.IsNullOrWhiteSpace(timestamp) && DateTime.TryParse(timestamp, out DateTime time)) { parser.registry.AddValueToLog(entry, LogAttribute.Timestamp, time); } var message = GetElementDataFromPath(parser.messagePath, element); if (message != null) { parser.registry.AddValueToLog(entry, LogAttribute.Message, message); } var result = ProcessCommonLogAttributes(parser, config, entry, element); parser.registry.NotifyFailedLogParsed(entry); return(result); }
public Task ParseAsync(Stream logStream, CancellationToken cancellationToken) { CheckUsability(); contextParseCounter.Increment(); return(XMLLogParser.InternalParse(parser, config, logStream, cancellationToken)); }
public void ApplyContextConfig(IDictionary <ContextConfigs, object> config, Action <ILogParser> context) { CheckUsability(); contextApplyContextConfigCounter.Increment(); XMLLogParser.InternalApplyContextConfig(parser, this.config, config, context); }
public XMLContextParser(XMLLogParser parser, TransientParserConfigs priorConfig) { this.parser = parser; this.config = priorConfig; }
/// <summary> /// Parse (async) a stream of data to get applicable logs. /// </summary> /// <param name="logStream">The stream of log data.</param> /// <param name="cancellationToken">A cancellation token that can be used to cancel the work</param> /// <returns>Task representing the parse operation.</returns> public Task ParseAsync(Stream logStream, CancellationToken cancellationToken) { parseCounter.Increment(); return(XMLLogParser.InternalParse(this, defaultTransientConfig, logStream, cancellationToken)); }
private static void ParseLoop(XMLLogParser parser, TransientParserConfigs config, Stream input, CancellationToken cancellationToken) { using (var xmlReader = XmlReader.Create(input)) { // Don't just use XML's natual tree-creation setup... because we don't want the root element to have children. For large or streaming logs, it will become a resource hog var elements = new Stack <XElement>(); bool exitLoop = false; while (!exitLoop && !cancellationToken.IsCancellationRequested && xmlReader.Read()) { switch (xmlReader.NodeType) { case XmlNodeType.Element: var name = XName.Get(xmlReader.Name, xmlReader.NamespaceURI); var element = new XElement(name); if (elements.Count > 1) // Don't add children to root to ensure it doesn't become massive { elements.Peek().Add(element); } elements.Push(element); if (xmlReader.HasAttributes) { while (xmlReader.MoveToNextAttribute()) { var attName = XName.Get(xmlReader.Name, xmlReader.NamespaceURI); var att = new XAttribute(attName, xmlReader.Value); elements.Peek().Add(att); } xmlReader.MoveToElement(); } break; case XmlNodeType.EndElement: xmlElementCounter.Increment(); if (xmlReader.Name == elements.Peek().Name) { var finishedElement = elements.Pop(); if (elements.Count == 1) // Don't process the elements unless they're children of root. Anything else is a child element of a log element { using (processElementTimer.NewContext()) { if (ProcessElement(parser, config, finishedElement) != LogParserErrors.OK) { exitLoop = true; break; } } } } else { Console.Error.WriteLine($"Element {elements.Peek().Name} ended, but the name of the ending element {xmlReader.Name} doesn't match. Possibly out of sync..."); } break; case XmlNodeType.CDATA: xmlCDATACounter.Increment(); elements.Peek().Add(new XCData(xmlReader.Value)); break; case XmlNodeType.Text: xmlTextCounter.Increment(); elements.Peek().Add(new XText(xmlReader.Value)); break; case XmlNodeType.Whitespace: break; default: unknownXmlElementCounter.Increment(xmlReader.NodeType.ToString()); break; } } // Only do if cancellation wasn't requested because it means we ran to completion but never found root. Cancellation means we never will. if (elements.Count != 0 && !cancellationToken.IsCancellationRequested) { xmlRootUnfinishedCounter.Increment(); Console.Error.WriteLine("Root element didn't end"); } } }