private static Task InternalParse(XMLLogParser parser, TransientParserConfigs config, Stream logStream, CancellationToken cancellationToken)
{
var parserStream =
!config.LogHasRoot ?
Encoding.UTF8.GetBytes("<root>").Append(logStream) :
logStream;
if (!config.LogHasRoot)
{
parserStream = parserStream.Append(Encoding.UTF8.GetBytes("</root>"));
}
var parseTask = Task.Run(() =>
{
try
{
ParseLoop(parser, config, parserStream, cancellationToken);
}
catch
{
//XXX probably want to do something here...
}
}, cancellationToken);
if (!config.LogHasRoot)
{
// If we didn't have the root, then we end up creating a Stream by appending streams together and need to dispose it
return(parseTask.ContinueWith(_ => parserStream.Dispose(), TaskContinuationOptions.ExecuteSynchronously));
}
return(parseTask);
}
private static LogParserErrors ProcessValidElement(XMLLogParser parser, TransientParserConfigs config, XElement element)
{
if (!parser.validConfigs)
{
return(LogParserErrors.ConfigNotInitialized);
}
if (parser.registry == null)
{
return(LogParserErrors.RegistryNotSet);
}
if (parser.timestampPath == null || parser.messagePath == null)
{
return(LogParserErrors.ConfigValueInvalid);
}
var timestamp = GetElementDataFromPath(parser.timestampPath, element);
if (string.IsNullOrWhiteSpace(timestamp))
{
return(LogParserErrors.MissingTimestamp);
}
var message = GetElementDataFromPath(parser.messagePath, element);
if (message == null)
{
return(LogParserErrors.MissingMessage);
}
var entry = parser.registry.AddLog(timestamp, message);
return(ProcessCommonLogAttributes(parser, config, entry, element));
}
private static LogParserErrors ProcessElement(XMLLogParser parser, TransientParserConfigs config, XElement element)
{
processElementCounter.Increment();
var result = ProcessValidElement(parser, config, element);
if (result != LogParserErrors.OK && !ParserUtil.IsFatal(result))
{
if (config.FailureHandling == LogParserFailureHandling.SkipEntries)
{
return(LogParserErrors.OK);
}
else if (config.FailureHandling == LogParserFailureHandling.MarkEntriesAsFailed)
{
return(ProcessInvalidElement(parser, config, element));
}
}
return(result);
}
private static LogParserErrors ProcessCommonLogAttributes(XMLLogParser parser, TransientParserConfigs config, ILogEntry entry, XElement element)
{
var attributesAdded = 0;
foreach (var kv in parser.attributePaths)
{
var rawValue = GetElementDataFromPath(kv.Value, element);
var fieldType = ParserPathElementFieldType.String;
if (kv.Value.Length != 0)
{
fieldType = kv.Value.Last().FieldType;
}
var value = ParserUtil.CastField(rawValue, fieldType);
if (value != null)
{
//XXX should probably have some test for checking if the value couldn't be added
if (parser.registry.AddValueToLog(entry, kv.Key, value))
{
attributesAdded++;
}
else
{
failureToAddAttributeToLogCounter.Increment(kv.Key.ToString());
}
}
}
if (config.AdditionalAttributes != null)
{
foreach (var kv in config.AdditionalAttributes)
{
if (parser.registry.AddValueToLog(entry, kv.Key, kv.Value))
{
attributesAdded++;
}
else
{
failureToAddAttributeToLogCounter.Increment(kv.Key.ToString());
}
}
}
attributesParsedHistogram.Update(attributesAdded);
return(LogParserErrors.OK);
}
private static void InternalApplyContextConfig(XMLLogParser parser, TransientParserConfigs priorConfig, IDictionary <ContextConfigs, object> config, Action <ILogParser> context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
var contextParser = new XMLContextParser(parser, priorConfig);
contextParser.ApplyConfig(config);
try
{
context(contextParser);
}
finally
{
contextParser.IsUsable = false;
}
}
private static LogParserErrors ProcessInvalidElement(XMLLogParser parser, TransientParserConfigs config, XElement element)
{
var entry = parser.registry.AddFailedLog();
var timestamp = GetElementDataFromPath(parser.timestampPath, element);
if (!string.IsNullOrWhiteSpace(timestamp) && DateTime.TryParse(timestamp, out DateTime time))
{
parser.registry.AddValueToLog(entry, LogAttribute.Timestamp, time);
}
var message = GetElementDataFromPath(parser.messagePath, element);
if (message != null)
{
parser.registry.AddValueToLog(entry, LogAttribute.Message, message);
}
var result = ProcessCommonLogAttributes(parser, config, entry, element);
parser.registry.NotifyFailedLogParsed(entry);
return(result);
}
public Task ParseAsync(Stream logStream, CancellationToken cancellationToken)
{
CheckUsability();
contextParseCounter.Increment();
return(XMLLogParser.InternalParse(parser, config, logStream, cancellationToken));
}
public void ApplyContextConfig(IDictionary <ContextConfigs, object> config, Action <ILogParser> context)
{
CheckUsability();
contextApplyContextConfigCounter.Increment();
XMLLogParser.InternalApplyContextConfig(parser, this.config, config, context);
}
public XMLContextParser(XMLLogParser parser, TransientParserConfigs priorConfig)
{
this.parser = parser;
this.config = priorConfig;
}
/// <summary>
/// Parse (async) a stream of data to get applicable logs.
/// </summary>
/// <param name="logStream">The stream of log data.</param>
/// <param name="cancellationToken">A cancellation token that can be used to cancel the work</param>
/// <returns>Task representing the parse operation.</returns>
public Task ParseAsync(Stream logStream, CancellationToken cancellationToken)
{
parseCounter.Increment();
return(XMLLogParser.InternalParse(this, defaultTransientConfig, logStream, cancellationToken));
}
private static void ParseLoop(XMLLogParser parser, TransientParserConfigs config, Stream input, CancellationToken cancellationToken)
{
using (var xmlReader = XmlReader.Create(input))
{
// Don't just use XML's natual tree-creation setup... because we don't want the root element to have children. For large or streaming logs, it will become a resource hog
var elements = new Stack <XElement>();
bool exitLoop = false;
while (!exitLoop && !cancellationToken.IsCancellationRequested && xmlReader.Read())
{
switch (xmlReader.NodeType)
{
case XmlNodeType.Element:
var name = XName.Get(xmlReader.Name, xmlReader.NamespaceURI);
var element = new XElement(name);
if (elements.Count > 1) // Don't add children to root to ensure it doesn't become massive
{
elements.Peek().Add(element);
}
elements.Push(element);
if (xmlReader.HasAttributes)
{
while (xmlReader.MoveToNextAttribute())
{
var attName = XName.Get(xmlReader.Name, xmlReader.NamespaceURI);
var att = new XAttribute(attName, xmlReader.Value);
elements.Peek().Add(att);
}
xmlReader.MoveToElement();
}
break;
case XmlNodeType.EndElement:
xmlElementCounter.Increment();
if (xmlReader.Name == elements.Peek().Name)
{
var finishedElement = elements.Pop();
if (elements.Count == 1) // Don't process the elements unless they're children of root. Anything else is a child element of a log element
{
using (processElementTimer.NewContext())
{
if (ProcessElement(parser, config, finishedElement) != LogParserErrors.OK)
{
exitLoop = true;
break;
}
}
}
}
else
{
Console.Error.WriteLine($"Element {elements.Peek().Name} ended, but the name of the ending element {xmlReader.Name} doesn't match. Possibly out of sync...");
}
break;
case XmlNodeType.CDATA:
xmlCDATACounter.Increment();
elements.Peek().Add(new XCData(xmlReader.Value));
break;
case XmlNodeType.Text:
xmlTextCounter.Increment();
elements.Peek().Add(new XText(xmlReader.Value));
break;
case XmlNodeType.Whitespace:
break;
default:
unknownXmlElementCounter.Increment(xmlReader.NodeType.ToString());
break;
}
}
// Only do if cancellation wasn't requested because it means we ran to completion but never found root. Cancellation means we never will.
if (elements.Count != 0 && !cancellationToken.IsCancellationRequested)
{
xmlRootUnfinishedCounter.Increment();
Console.Error.WriteLine("Root element didn't end");
}
}
}
