private string GetHost(MessageDataItem msg)
{
try
{
return(msg.GetAttributeAsString(outputConfiguration.FieldMappings.HostAttribute));
}
catch
{
if (outputConfiguration.EventMetadataDefaults.Host != null)
{
return(outputConfiguration.EventMetadataDefaults.Host);
}
ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to get host from message and no default host, using localhost instead.");
return("localhost");
}
}
private string GetSource(MessageDataItem msg)
{
try
{
return(msg.GetAttributeAsString(outputConfiguration.FieldMappings.SourceAttribute));
}
catch
{
if (outputConfiguration.EventMetadataDefaults.Source != null)
{
return(outputConfiguration.EventMetadataDefaults.Source);
}
ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to get source from message and no default source set, using none instead.");
return("none");
}
}
private string GetIndex(MessageDataItem msg)
{
try
{
return(msg.GetAttributeAsString(outputConfiguration.FieldMappings.IndexAttribute));
}
catch
{
if (outputConfiguration.EventMetadataDefaults.Index != null)
{
return(outputConfiguration.EventMetadataDefaults.Index);
}
ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to get index from message and no default index set, using main index instead.");
return("main");
}
}
private string GetEPOCHTime(MessageDataItem msg)
{
if (outputConfiguration.EventMetadataDefaults.UseCurrentTime)
{
return(DateTime.UtcNow.Subtract(epoch).TotalSeconds.ToString("F3"));
}
try
{
DateTime msgTime = msg.GetAttributeAsDateTime(outputConfiguration.FieldMappings.TimeAttribute);
if (msgTime.Kind == DateTimeKind.Local)
{
msgTime = msgTime.ToUniversalTime();
}
return(msgTime.Subtract(epoch).TotalSeconds.ToString("F3"));
}
catch
{
ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to get time from message, using current time instead.");
return(DateTime.Now.Subtract(epoch).TotalSeconds.ToString("F3"));
}
}
public void ExtractAttributes(MessageDataItem message)
{
try
{
if (message.Message.IndexOf('=') == -1)
{
return;
}
// start state machine
int pos = 0;
int len = message.Message.Length;
while (pos < len)
{
pos++;
}
return;
}
catch (Exception e)
{
ServerLogger?.LogEvent(this, Severity.Warning, "KVExtractor", "Error while extracting key-value pairs.", e);
}
}
private void WorkerProc()
{
DateTime start = DateTime.Now;
int counter = 0;
int messageBatchCounter = 0;
StringBuilder outputBuffer = new StringBuilder(maxBufferSize + maxBufferSize / 10); // 110%
void FlushOutputBuffer(bool lastSubmission)
{
if (outputBuffer.Length == 0)
{
return;
}
bool sent = false;
while (!sent)
{
try
{
if (token.IsCancellationRequested && !lastSubmission)
{
break;
}
Task submitTask = SubmitHECEvent(outputBuffer);
if (lastSubmission)
{
submitTask.Wait(5 * 1000); // 5 seconds
}
else
{
submitTask.Wait(token);
}
sent = true;
break;
}
catch (Exception e)
{
sent = false;
ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to send request.", e);
if (lastSubmission)
{
break;
}
continue;
}
}
outputBuffer.Clear();
messageBatchCounter = 0;
}
/*
* Buffer flush rules:
* 1. Flush if buffer size is over maxBufferSize, OR, if buffer contains mother than maxBatchSize messages
* 2. If there is no messages in the input queue left
*/
while (true)
{
if (token.IsCancellationRequested)
{
FlushOutputBuffer(lastSubmission: true);
break;
}
if (MessageReceiver.Invoke(out MessageDataItem newMessage))
{
try
{
messageBatchCounter++;
// root object & meta
JObject hecRequestBody = new JObject
{
{ "time", GetEPOCHTime(newMessage) },
{ "host", GetHost(newMessage) },
{ "source", GetSource(newMessage) },
{ "sourcetype", GetSourcetype(newMessage) },
{ "index", GetIndex(newMessage) }
};
// event other attributes
JObject whereToAddAttributes;
if (outputConfiguration.UseFields)
{
whereToAddAttributes = new JObject();
hecRequestBody.Add("event", newMessage.Message);
}
else
{
whereToAddAttributes = new JObject()
{
{ "Message", newMessage.Message }
}
};
foreach (string attrName in newMessage.GetAttributeNames)
{
Variant attrValue = newMessage.GetAttributeAsVariant(attrName);
switch (attrValue.Type)
{
case VariantType.Boolean:
whereToAddAttributes.Add(attrName, attrValue.BooleanValue);
break;
case VariantType.DateTime:
whereToAddAttributes.Add(attrName, attrValue.DateTimeValue);
break;
case VariantType.Float:
whereToAddAttributes.Add(attrName, attrValue.FloatValue);
break;
case VariantType.Int:
whereToAddAttributes.Add(attrName, attrValue.IntValue);
break;
case VariantType.String:
whereToAddAttributes.Add(attrName, attrValue.StringValue);
break;
}
}
if (outputConfiguration.UseFields)
{
hecRequestBody.Add("fields", whereToAddAttributes);
}
else
{
hecRequestBody.Add("event", whereToAddAttributes);
}
outputBuffer.AppendLine(hecRequestBody.ToString());
if (messageBatchCounter >= maxBatchSize || outputBuffer.Length >= maxBufferSize)
{
FlushOutputBuffer(lastSubmission: false);
}
// report performance
counter++;
if (counter > 100000)
{
double msgps = counter / DateTime.Now.Subtract(start).TotalSeconds;
Console.WriteLine($"Rate: {msgps:N2} msg/sec.");
ServerHealthReporter.SetPerformanceCounter(this, null, "MsgPerSecond", msgps);
start = DateTime.Now;
counter = 0;
}
}
catch (Exception e)
{
ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to create request.", e);
}
}
else
{
FlushOutputBuffer(lastSubmission: false);
Thread.Sleep(DefaultIdleDelay);
}
}
}
public MessageDataItem Parse(MessageDataItem message)
{
// result -- keep it as separate vars to allow modifications and overrides
string outMessage = null;
Dictionary <string, Variant> outAttributes = new Dictionary <string, Variant>();
bool AnyMatches = false;
if (config.DefaultFieldSettings != null)
{
foreach (FieldSetting attrCfg in config.DefaultFieldSettings)
{
try
{
attrCfg.SetField(null, message, ref outMessage, outAttributes);
}
catch (Exception e)
{
ServerLogger?.LogEvent(this, Severity.Warning, "Parser", $"Failed to parse field {attrCfg.OutputAttribute ?? "<UNKNOWN>"}", e);
}
}
}
foreach (ParsingExpression expr in config.ParsingExpressions)
{
if (string.IsNullOrEmpty(expr.MatchingRegEx) || Regex.IsMatch(message.Message, expr.MatchingRegEx))
{
MatchCollection matches = Regex.Matches(message.Message, expr.ParsingRegEx, expr.RegexOptions);
if (matches.Count > 0)
{
AnyMatches = true;
foreach (Match match in matches)
{
if (match.Success)
{
if (expr.FieldSettings != null)
{
foreach (FieldSetting attrCfg in expr.FieldSettings)
{
try
{
attrCfg.SetField(match.Groups, message, ref outMessage, outAttributes);
}
catch (Exception e)
{
ServerLogger?.LogEvent(this, Severity.Warning, "Parser", $"Failed to parse field {attrCfg.OutputAttribute ?? "<UNKNOWN>"}", e);
}
}
}
}
}
}
else
{
if (expr.StopIfMatched)
{
ServerLogger?.LogEvent(this, Severity.Warning, "Parser", "Matching expression matched in a stopping expression, but the parsing expression didn't match.");
}
}
// exit from loop of ParsingExpressions
if (expr.StopIfMatched)
{
break;
}
}
}
if (AnyMatches)
{
MessageDataItem result = new MessageDataItem(outMessage ?? "");
foreach (KeyValuePair <string, Variant> newAttribute in outAttributes)
{
result.AddAttribute(newAttribute.Key, newAttribute.Value);
}
return(result);
}
else
{
if (config.PassthroughFieldSettings != null)
{
foreach (FieldSetting attrCfg in config.PassthroughFieldSettings)
{
try
{
attrCfg.SetField(null, message, ref outMessage, outAttributes);
}
catch (Exception e)
{
ServerLogger?.LogEvent(this, Severity.Warning, "Parser", $"Failed to parse field {attrCfg.OutputAttribute ?? "<UNKNOWN>"}", e);
}
}
}
MessageDataItem result = new MessageDataItem(outMessage ?? "");
foreach (KeyValuePair <string, Variant> newAttribute in outAttributes)
{
result.AddAttribute(newAttribute.Key, newAttribute.Value);
}
return(result);
// return message.Clone();
}
}
