private void WorkerProc()
{
DateTime start = DateTime.Now;
int counter = 0;
int messageBatchCounter = 0;
StringBuilder outputBuffer = new StringBuilder(maxBufferSize + maxBufferSize / 10); // 110%
void FlushOutputBuffer(bool lastSubmission)
{
if (outputBuffer.Length == 0)
{
return;
}
bool sent = false;
while (!sent)
{
try
{
if (token.IsCancellationRequested && !lastSubmission)
{
break;
}
Task submitTask = SubmitHECEvent(outputBuffer);
if (lastSubmission)
{
submitTask.Wait(5 * 1000); // 5 seconds
}
else
{
submitTask.Wait(token);
}
sent = true;
break;
}
catch (Exception e)
{
sent = false;
ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to send request.", e);
if (lastSubmission)
{
break;
}
continue;
}
}
outputBuffer.Clear();
messageBatchCounter = 0;
}
/*
* Buffer flush rules:
* 1. Flush if buffer size is over maxBufferSize, OR, if buffer contains mother than maxBatchSize messages
* 2. If there is no messages in the input queue left
*/
while (true)
{
if (token.IsCancellationRequested)
{
FlushOutputBuffer(lastSubmission: true);
break;
}
if (MessageReceiver.Invoke(out MessageDataItem newMessage))
{
try
{
messageBatchCounter++;
// root object & meta
JObject hecRequestBody = new JObject
{
{ "time", GetEPOCHTime(newMessage) },
{ "host", GetHost(newMessage) },
{ "source", GetSource(newMessage) },
{ "sourcetype", GetSourcetype(newMessage) },
{ "index", GetIndex(newMessage) }
};
// event other attributes
JObject whereToAddAttributes;
if (outputConfiguration.UseFields)
{
whereToAddAttributes = new JObject();
hecRequestBody.Add("event", newMessage.Message);
}
else
{
whereToAddAttributes = new JObject()
{
{ "Message", newMessage.Message }
}
};
foreach (string attrName in newMessage.GetAttributeNames)
{
Variant attrValue = newMessage.GetAttributeAsVariant(attrName);
switch (attrValue.Type)
{
case VariantType.Boolean:
whereToAddAttributes.Add(attrName, attrValue.BooleanValue);
break;
case VariantType.DateTime:
whereToAddAttributes.Add(attrName, attrValue.DateTimeValue);
break;
case VariantType.Float:
whereToAddAttributes.Add(attrName, attrValue.FloatValue);
break;
case VariantType.Int:
whereToAddAttributes.Add(attrName, attrValue.IntValue);
break;
case VariantType.String:
whereToAddAttributes.Add(attrName, attrValue.StringValue);
break;
}
}
if (outputConfiguration.UseFields)
{
hecRequestBody.Add("fields", whereToAddAttributes);
}
else
{
hecRequestBody.Add("event", whereToAddAttributes);
}
outputBuffer.AppendLine(hecRequestBody.ToString());
if (messageBatchCounter >= maxBatchSize || outputBuffer.Length >= maxBufferSize)
{
FlushOutputBuffer(lastSubmission: false);
}
// report performance
counter++;
if (counter > 100000)
{
double msgps = counter / DateTime.Now.Subtract(start).TotalSeconds;
Console.WriteLine($"Rate: {msgps:N2} msg/sec.");
ServerHealthReporter.SetPerformanceCounter(this, null, "MsgPerSecond", msgps);
start = DateTime.Now;
counter = 0;
}
}
catch (Exception e)
{
ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to create request.", e);
}
}
else
{
FlushOutputBuffer(lastSubmission: false);
Thread.Sleep(DefaultIdleDelay);
}
}
}
private void WorkerProcCTSV()
{
DateTime start = DateTime.Now;
int counter = 0;
int messageBatchCounter = 0;
StringBuilder outputBuffer = new StringBuilder(maxBufferSize + maxBufferSize / 10); // 110%
void FlushOutputBuffer()
{
if (outputBuffer.Length == 0)
{
return;
}
File.AppendAllText(outputConfiguration.GetFileName(), outputBuffer.ToString(), fileEncoding);
outputBuffer.Clear();
messageBatchCounter = 0;
}
/*
* Buffer flush rules:
* 1. Flush if buffer size is over maxBufferSize, OR, if buffer contains mother than maxBatchSize messages
* 2. If there is no messages in the input queue left
*/
while (true)
{
if (token.IsCancellationRequested)
{
FlushOutputBuffer();
break;
}
if (MessageReceiver.Invoke(out MessageDataItem newMessage))
{
messageBatchCounter++;
if (!string.IsNullOrEmpty(newMessage.Message))
{
outputBuffer.Append(QuoteString(newMessage.Message));
}
foreach (string attrName in newMessage.GetAttributeNames)
{
outputBuffer.Append(separator);
Variant attrValue = newMessage.GetAttributeAsVariant(attrName);
switch (attrValue.Type)
{
case VariantType.Boolean:
outputBuffer.Append(QuoteString(attrValue.BooleanValue.ToString()));
break;
case VariantType.DateTime:
outputBuffer.Append(QuoteString(attrValue.DateTimeValue.ToString("o")));
break;
case VariantType.Float:
outputBuffer.Append(QuoteString(attrValue.FloatValue.ToString()));
break;
case VariantType.Int:
outputBuffer.Append(QuoteString(attrValue.IntValue.ToString()));
break;
case VariantType.String:
outputBuffer.Append(QuoteString(attrValue.StringValue ?? ""));
break;
}
}
outputBuffer.AppendLine();
if (messageBatchCounter >= maxBatchSize || outputBuffer.Length >= maxBufferSize)
{
FlushOutputBuffer();
}
// report performance
counter++;
if (counter > 100000)
{
double msgps = counter / DateTime.Now.Subtract(start).TotalSeconds;
ServerHealthReporter.SetPerformanceCounter(this, null, "MsgPerSecond", msgps);
start = DateTime.Now;
counter = 0;
}
}
else
{
// flush any outstanding data if nothing else to do
FlushOutputBuffer();
Thread.Sleep(DefaultIdleDelay);
}
}
}
