private string GetHost(MessageDataItem msg) { try { return(msg.GetAttributeAsString(outputConfiguration.FieldMappings.HostAttribute)); } catch { if (outputConfiguration.EventMetadataDefaults.Host != null) { return(outputConfiguration.EventMetadataDefaults.Host); } ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to get host from message and no default host, using localhost instead."); return("localhost"); } }
private string GetSource(MessageDataItem msg) { try { return(msg.GetAttributeAsString(outputConfiguration.FieldMappings.SourceAttribute)); } catch { if (outputConfiguration.EventMetadataDefaults.Source != null) { return(outputConfiguration.EventMetadataDefaults.Source); } ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to get source from message and no default source set, using none instead."); return("none"); } }
private string GetIndex(MessageDataItem msg) { try { return(msg.GetAttributeAsString(outputConfiguration.FieldMappings.IndexAttribute)); } catch { if (outputConfiguration.EventMetadataDefaults.Index != null) { return(outputConfiguration.EventMetadataDefaults.Index); } ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to get index from message and no default index set, using main index instead."); return("main"); } }
private string GetEPOCHTime(MessageDataItem msg) { if (outputConfiguration.EventMetadataDefaults.UseCurrentTime) { return(DateTime.UtcNow.Subtract(epoch).TotalSeconds.ToString("F3")); } try { DateTime msgTime = msg.GetAttributeAsDateTime(outputConfiguration.FieldMappings.TimeAttribute); if (msgTime.Kind == DateTimeKind.Local) { msgTime = msgTime.ToUniversalTime(); } return(msgTime.Subtract(epoch).TotalSeconds.ToString("F3")); } catch { ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to get time from message, using current time instead."); return(DateTime.Now.Subtract(epoch).TotalSeconds.ToString("F3")); } }
public void ExtractAttributes(MessageDataItem message) { try { if (message.Message.IndexOf('=') == -1) { return; } // start state machine int pos = 0; int len = message.Message.Length; while (pos < len) { pos++; } return; } catch (Exception e) { ServerLogger?.LogEvent(this, Severity.Warning, "KVExtractor", "Error while extracting key-value pairs.", e); } }
private void WorkerProc() { DateTime start = DateTime.Now; int counter = 0; int messageBatchCounter = 0; StringBuilder outputBuffer = new StringBuilder(maxBufferSize + maxBufferSize / 10); // 110% void FlushOutputBuffer(bool lastSubmission) { if (outputBuffer.Length == 0) { return; } bool sent = false; while (!sent) { try { if (token.IsCancellationRequested && !lastSubmission) { break; } Task submitTask = SubmitHECEvent(outputBuffer); if (lastSubmission) { submitTask.Wait(5 * 1000); // 5 seconds } else { submitTask.Wait(token); } sent = true; break; } catch (Exception e) { sent = false; ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to send request.", e); if (lastSubmission) { break; } continue; } } outputBuffer.Clear(); messageBatchCounter = 0; } /* * Buffer flush rules: * 1. Flush if buffer size is over maxBufferSize, OR, if buffer contains mother than maxBatchSize messages * 2. If there is no messages in the input queue left */ while (true) { if (token.IsCancellationRequested) { FlushOutputBuffer(lastSubmission: true); break; } if (MessageReceiver.Invoke(out MessageDataItem newMessage)) { try { messageBatchCounter++; // root object & meta JObject hecRequestBody = new JObject { { "time", GetEPOCHTime(newMessage) }, { "host", GetHost(newMessage) }, { "source", GetSource(newMessage) }, { "sourcetype", GetSourcetype(newMessage) }, { "index", GetIndex(newMessage) } }; // event other attributes JObject whereToAddAttributes; if (outputConfiguration.UseFields) { whereToAddAttributes = new JObject(); hecRequestBody.Add("event", newMessage.Message); } else { whereToAddAttributes = new JObject() { { "Message", newMessage.Message } } }; foreach (string attrName in newMessage.GetAttributeNames) { Variant attrValue = newMessage.GetAttributeAsVariant(attrName); switch (attrValue.Type) { case VariantType.Boolean: whereToAddAttributes.Add(attrName, attrValue.BooleanValue); break; case VariantType.DateTime: whereToAddAttributes.Add(attrName, attrValue.DateTimeValue); break; case VariantType.Float: whereToAddAttributes.Add(attrName, attrValue.FloatValue); break; case VariantType.Int: whereToAddAttributes.Add(attrName, attrValue.IntValue); break; case VariantType.String: whereToAddAttributes.Add(attrName, attrValue.StringValue); break; } } if (outputConfiguration.UseFields) { hecRequestBody.Add("fields", whereToAddAttributes); } else { hecRequestBody.Add("event", whereToAddAttributes); } outputBuffer.AppendLine(hecRequestBody.ToString()); if (messageBatchCounter >= maxBatchSize || outputBuffer.Length >= maxBufferSize) { FlushOutputBuffer(lastSubmission: false); } // report performance counter++; if (counter > 100000) { double msgps = counter / DateTime.Now.Subtract(start).TotalSeconds; Console.WriteLine($"Rate: {msgps:N2} msg/sec."); ServerHealthReporter.SetPerformanceCounter(this, null, "MsgPerSecond", msgps); start = DateTime.Now; counter = 0; } } catch (Exception e) { ServerLogger?.LogEvent(this, Severity.Warning, "SplunkHEC", "Failed to create request.", e); } } else { FlushOutputBuffer(lastSubmission: false); Thread.Sleep(DefaultIdleDelay); } } }
public MessageDataItem Parse(MessageDataItem message) { // result -- keep it as separate vars to allow modifications and overrides string outMessage = null; Dictionary <string, Variant> outAttributes = new Dictionary <string, Variant>(); bool AnyMatches = false; if (config.DefaultFieldSettings != null) { foreach (FieldSetting attrCfg in config.DefaultFieldSettings) { try { attrCfg.SetField(null, message, ref outMessage, outAttributes); } catch (Exception e) { ServerLogger?.LogEvent(this, Severity.Warning, "Parser", $"Failed to parse field {attrCfg.OutputAttribute ?? "<UNKNOWN>"}", e); } } } foreach (ParsingExpression expr in config.ParsingExpressions) { if (string.IsNullOrEmpty(expr.MatchingRegEx) || Regex.IsMatch(message.Message, expr.MatchingRegEx)) { MatchCollection matches = Regex.Matches(message.Message, expr.ParsingRegEx, expr.RegexOptions); if (matches.Count > 0) { AnyMatches = true; foreach (Match match in matches) { if (match.Success) { if (expr.FieldSettings != null) { foreach (FieldSetting attrCfg in expr.FieldSettings) { try { attrCfg.SetField(match.Groups, message, ref outMessage, outAttributes); } catch (Exception e) { ServerLogger?.LogEvent(this, Severity.Warning, "Parser", $"Failed to parse field {attrCfg.OutputAttribute ?? "<UNKNOWN>"}", e); } } } } } } else { if (expr.StopIfMatched) { ServerLogger?.LogEvent(this, Severity.Warning, "Parser", "Matching expression matched in a stopping expression, but the parsing expression didn't match."); } } // exit from loop of ParsingExpressions if (expr.StopIfMatched) { break; } } } if (AnyMatches) { MessageDataItem result = new MessageDataItem(outMessage ?? ""); foreach (KeyValuePair <string, Variant> newAttribute in outAttributes) { result.AddAttribute(newAttribute.Key, newAttribute.Value); } return(result); } else { if (config.PassthroughFieldSettings != null) { foreach (FieldSetting attrCfg in config.PassthroughFieldSettings) { try { attrCfg.SetField(null, message, ref outMessage, outAttributes); } catch (Exception e) { ServerLogger?.LogEvent(this, Severity.Warning, "Parser", $"Failed to parse field {attrCfg.OutputAttribute ?? "<UNKNOWN>"}", e); } } } MessageDataItem result = new MessageDataItem(outMessage ?? ""); foreach (KeyValuePair <string, Variant> newAttribute in outAttributes) { result.AddAttribute(newAttribute.Key, newAttribute.Value); } return(result); // return message.Clone(); } }