private void DoSerialization(HttpEventCollectorEventInfo eventInfo) { string serializedEventInfo; if (formatter == null) { serializedEventInfo = JsonConvert.SerializeObject(eventInfo); } else { var formattedEvent = formatter(eventInfo); eventInfo.Event = formattedEvent; serializedEventInfo = JsonConvert.SerializeObject(eventInfo); } lock (eventsBatchLock) { eventsBatch.Add(eventInfo); serializedEventsBatch.Append(serializedEventInfo); if (eventsBatch.Count >= batchSizeCount || serializedEventsBatch.Length >= batchSizeBytes) { FlushInternal(); } } }
private void DoSerialization(HttpEventCollectorEventInfo ei) { string serializedEventInfo; if (formatter == null) { serializedEventInfo = SerializeEventInfo(ei); } else { var formattedEvent = formatter(ei); ei.Event = formattedEvent; serializedEventInfo = JsonConvert.SerializeObject(ei); } // we use lock serializedEventsBatch to synchronize both // serializedEventsBatch and serializedEvents lock (eventsBatchLock) { eventsBatch.Add(ei); serializedEventsBatch.Append(serializedEventInfo); if (eventsBatch.Count >= batchSizeCount || serializedEventsBatch.Length >= batchSizeBytes) { // there are enough events in the batch FlushInternal(); } } }
public void HttpEventCollectorBatchingSizeTest() { // estimate serialized event size HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(null, TraceEventType.Information.ToString(), "info ?", null, null); int size = HttpEventCollectorSender.SerializeEventInfo(ei).Length; var trace = Trace( handler: (token, events) => { Assert.True(events.Count == 4); Assert.True(events[0].Event.Message == "info 1"); Assert.True(events[1].Event.Message == "info 2"); Assert.True(events[2].Event.Message == "info 3"); Assert.True(events[3].Event.Message == "info 4"); return(new Response()); }, batchSizeBytes: 4 * size - size / 2 // 4 events trigger post ); trace.TraceInformation("info 1"); trace.TraceInformation("info 2"); trace.TraceInformation("info 3"); trace.TraceInformation("info 4"); trace.TraceInformation("info 1"); trace.TraceInformation("info 2"); trace.TraceInformation("info 3"); trace.TraceInformation("info 4"); trace.Close(); }
/// <summary> /// Does the serialization. /// </summary> /// <param name="ei">The ei.</param> private void DoSerialization(HttpEventCollectorEventInfo ei) { if (formatter != null) { var formattedEvent = formatter(ei); ei.Event = formattedEvent; } long orgLength = serializedEventsBatch.BaseStream.Length; try { using (JsonTextWriter jsonWriter = new JsonTextWriter(serializedEventsBatch)) { jsonWriter.Formatting = jsonSerializer.Formatting; jsonSerializer.Serialize(jsonWriter, ei); } serializedEventsBatch.Flush(); } catch { // Unwind / truncate any bad output serializedEventsBatch.Flush(); serializedEventsBatch.BaseStream.Position = orgLength; serializedEventsBatch.BaseStream.SetLength(orgLength); jsonSerializer = JsonSerializer.CreateDefault(jsonSerializerSettings); // Reset bad state throw; } }
/// <summary> /// Send an event to Splunk HTTP endpoint. Actual event send is done /// asynchronously and this method doesn't block client application. /// </summary> /// <param name="id">Event id.</param> /// <param name="severity">Event severity info.</param> /// <param name="message">Event message text.</param> /// <param name="data">Additional event data.</param> public void Send( string id = null, string severity = null, string message = null, object data = null) { HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(id, severity, message, data, metadata); DoSerialization(ei); }
/// <summary> /// Send an event to Splunk HTTP endpoint. Actual event send is done /// asynchronously and this method doesn't block client application. /// </summary> /// <param name="timestamp">Timestamp to use.</param> /// <param name="id">Event id.</param> /// <param name="severity">Event severity info.</param> /// <param name="message">Event message text.</param> /// <param name="data">Additional event data.</param> /// <param name="metadataOverride">Metadata to use for this send.</param> public void Send( DateTime timestamp, string id = null, string severity = null, string message = null, object data = null, HttpEventCollectorEventInfo.Metadata metadataOverride = null) { HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(timestamp, id, severity, message, data, metadataOverride ?? metadata); DoSerialization(ei); }
/// <summary> /// Send an event to Splunk HTTP endpoint. Actual event send is done /// asynchronously and this method doesn't block client application. /// </summary> /// <param name="id">Event id.</param> /// <param name="level">Event severity info.</param> /// <param name="messageTemplate">Event message template.</param> /// <param name="renderedMessage">Event message rendered.</param> /// <param name="exception">Event exception.</param> /// <param name="properties">Additional event data.</param> /// <param name="metadataOverride">Metadata to use for this send.</param> public void Send( string id = null, string level = null, string messageTemplate = null, string renderedMessage = null, object exception = null, object properties = null, HttpEventCollectorEventInfo.Metadata metadataOverride = null) { HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(id, level, messageTemplate, renderedMessage, exception, properties, metadataOverride ?? metadata); DoSerialization(ei); }
private void DoSerialization(HttpEventCollectorEventInfo ei) { // If data contains valid Json deserialize to avoid double serialization if (ei.Event.Properties != null) { var properties = (Dictionary <String, object>)ei.Event.Properties; var keys = properties.Keys.ToList(); foreach (var key in keys) { var data = properties[key]; if (data != null && IsValidJson(data.ToString())) { properties[key] = JsonConvert.DeserializeObject <object>(data.ToString()); } } } string serializedEventInfo; if (formatter == null) { serializedEventInfo = SerializeEventInfo(ei); } else { var formattedEvent = formatter(ei); ei.Event = formattedEvent; serializedEventInfo = JsonConvert.SerializeObject(ei); } // we use lock serializedEventsBatch to synchronize both // serializedEventsBatch and serializedEvents lock (eventsBatchLock) { eventsBatch.Add(ei); serializedEventsBatch.Append(serializedEventInfo); if (eventsBatch.Count >= batchSizeCount || serializedEventsBatch.Length >= batchSizeBytes) { // there are enough events in the batch FlushInternal(); } } }
private void DoSerialization(HttpEventCollectorEventInfo ei) { if (formatter != null) { var formattedEvent = formatter(ei); ei.Event = formattedEvent; } // we use lock serializedEventsBatch to synchronize both // serializedEventsBatch and serializedEvents lock (eventsBatchLock) { ++eventsBatchCount; long orgLength = this.serializedEventsBatch.BaseStream.Length; try { using (JsonTextWriter jsonWriter = new JsonTextWriter(this.serializedEventsBatch)) { jsonWriter.Formatting = this.jsonSerializer.Formatting; this.jsonSerializer.Serialize(jsonWriter, ei); } this.serializedEventsBatch.Flush(); } catch { // Unwind / truncate any bad output this.serializedEventsBatch.Flush(); this.serializedEventsBatch.BaseStream.Position = orgLength; this.serializedEventsBatch.BaseStream.SetLength(orgLength); this.jsonSerializer = JsonSerializer.CreateDefault(this.jsonSerializerSettings); // Reset bad state throw; } if (eventsBatchCount >= batchSizeCount || serializedEventsBatch.BaseStream.Length >= batchSizeBytes) { // there are enough events in the batch FlushInternal(); } } }
public void HttpEventCollectorEventInfoTimestampsInvariantTest() { // Change Culture temporarily for this test var backupCulture = Thread.CurrentThread.CurrentCulture; Thread.CurrentThread.CurrentCulture = new CultureInfo("de-DE"); // test setting timestamp DateTime utcNow = DateTime.UtcNow; double nowEpoch = (utcNow - new DateTime(1970, 1, 1)).TotalSeconds; HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(utcNow.AddHours(-1), null, null, null, null, null); // Reset the culture before any assertions Thread.CurrentThread.CurrentCulture = backupCulture; // Ensure we have a comma when using a non-US culture Assert.True(ei.Timestamp.Contains(",")); }
public void HttpEventCollectorEventInfoTimestampsTest() { // test setting timestamp DateTime utcNow = DateTime.UtcNow; double nowEpoch = (utcNow - new DateTime(1970, 1, 1)).TotalSeconds; HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(utcNow.AddHours(-1), null, null, null, null, null); double epochTimestamp = double.Parse(ei.Timestamp); double diff = Math.Ceiling(nowEpoch - epochTimestamp); Assert.True(diff >= 3600.0); // test default timestamp ei = new HttpEventCollectorEventInfo(null, null, null, null, null); utcNow = DateTime.UtcNow; nowEpoch = (utcNow - new DateTime(1970, 1, 1)).TotalSeconds; epochTimestamp = double.Parse(ei.Timestamp); diff = Math.Ceiling(nowEpoch - epochTimestamp); Assert.True(diff < 10.0); }
/// <summary> /// Send an event to Splunk HTTP endpoint. Actual event send is done /// asynchronously and this method doesn't block client application. /// </summary> /// <param name="logData">log data to use for this send.</param> public void Send(object logData) { HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(logData); DoSerialization(ei); }
public void HttpEventCollectorBatchingSizeTest() { // estimate serialized event size HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(null, TraceEventType.Information.ToString(), "info ?", null, null); int size = HttpEventCollectorSender.SerializeEventInfo(ei).Length; var trace = Trace( handler: (token, events) => { Assert.True(events.Count == 4); Assert.True(events[0].Event.Message == "info 1"); Assert.True(events[1].Event.Message == "info 2"); Assert.True(events[2].Event.Message == "info 3"); Assert.True(events[3].Event.Message == "info 4"); return new Response(); }, batchSizeBytes: 4 * size - size / 2 // 4 events trigger post ); trace.TraceInformation("info 1"); trace.TraceInformation("info 2"); trace.TraceInformation("info 3"); trace.TraceInformation("info 4"); trace.TraceInformation("info 1"); trace.TraceInformation("info 2"); trace.TraceInformation("info 3"); trace.TraceInformation("info 4"); trace.Close(); }
/// <summary> /// Serialize event info into a json string /// </summary> /// <param name="eventInfo"></param> /// <returns></returns> public static string SerializeEventInfo(HttpEventCollectorEventInfo eventInfo) { return(JsonConvert.SerializeObject(eventInfo)); }
// Input trace listener private TraceSource Trace( RequestHandler handler, HttpEventCollectorEventInfo.Metadata metadata = null, int batchInterval = 0, int batchSizeBytes = 0, int batchSizeCount = 0, HttpEventCollectorSender.SendMode sendMode = HttpEventCollectorSender.SendMode.Parallel, HttpEventCollectorSender.HttpEventCollectorMiddleware middleware = null) { var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; trace.Listeners.Add( new HttpEventCollectorTraceListener( uri: uri, token: token, metadata: metadata, sendMode: sendMode, batchInterval: batchInterval, batchSizeBytes: batchSizeBytes, batchSizeCount: batchSizeCount, middleware: MiddlewareInterceptor(handler, middleware)) ); return trace; }
private SinkTrace TraceSource( RequestHandler handler, HttpEventCollectorEventInfo.Metadata metadata = null, HttpEventCollectorSender.SendMode sendMode = HttpEventCollectorSender.SendMode.Parallel, int batchInterval = 0, int batchSizeBytes = 0, int batchSizeCount = 0, HttpEventCollectorSender.HttpEventCollectorMiddleware middleware = null) { var listener = new ObservableEventListener(); var sink = new HttpEventCollectorSink( uri: uri, token: token, formatter: new TestEventFormatter(), metadata: metadata, sendMode: sendMode, batchInterval: batchInterval, batchSizeBytes: batchSizeBytes, batchSizeCount: batchSizeCount, middleware: MiddlewareInterceptor(handler, middleware)); listener.Subscribe(sink); var eventSource = TestEventSource.GetInstance(); listener.EnableEvents(eventSource, EventLevel.LogAlways, Keywords.All); return new SinkTrace() { Source = eventSource, Sink = sink, Listener = listener }; }
/// <param name="uri">Splunk server uri, for example https://localhost:8089.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="batchInterval">Batch interval in milliseconds.</param> /// <param name="batchSizeBytes">Batch max size.</param> /// <param name="batchSizeCount">MNax number of individual events in batch.</param> /// <param name="middleware"> /// HTTP client middleware. This allows to plug an HttpClient handler that /// intercepts logging HTTP traffic. /// </param> /// <remarks> /// Zero values for the batching params mean that batching is off. /// </remarks> public HttpEventCollectorSender( Uri uri, string token, HttpEventCollectorEventInfo.Metadata metadata, SendMode sendMode, int batchInterval, int batchSizeBytes, int batchSizeCount, HttpEventCollectorMiddleware middleware) { this.httpEventCollectorEndpointUri = new Uri(uri, HttpEventCollectorPath); this.sendMode = sendMode; this.batchInterval = batchInterval; this.batchSizeBytes = batchSizeBytes; this.batchSizeCount = batchSizeCount; this.metadata = metadata; this.token = token; this.middleware = middleware; // special case - if batch interval is specified without size and count // they are set to "infinity", i.e., batch may have any size if (this.batchInterval > 0 && this.batchSizeBytes == 0 && this.batchSizeCount == 0) { this.batchSizeBytes = this.batchSizeCount = int.MaxValue; } // when size configuration setting is missing it's treated as "infinity", // i.e., any value is accepted. if (this.batchSizeCount == 0 && this.batchSizeBytes > 0) { this.batchSizeCount = int.MaxValue; } else if (this.batchSizeBytes == 0 && this.batchSizeCount > 0) { this.batchSizeBytes = int.MaxValue; } // setup the timer if (batchInterval != 0) // 0 means - no timer { timer = new Timer(OnTimer, null, batchInterval, batchInterval); } // setup HTTP client httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(AuthorizationHeaderScheme, token); }
/// <summary> /// HttpEventCollectorTraceListener c-or. Instantiates HttpEventCollectorTraceListener /// when retriesOnError parameter is specified. /// </summary> /// <param name="uri">Splunk server uri, for example https://localhost:8089.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="retriesOnError">Number of retries when network problem is detected</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="batchInterval">Batch interval in milliseconds.</param> /// <param name="batchSizeBytes">Batch max size.</param> /// <param name="batchSizeCount">MNax number of individual events in batch.</param> public HttpEventCollectorTraceListener( Uri uri, string token, int retriesOnError, HttpEventCollectorEventInfo.Metadata metadata = null, HttpEventCollectorSender.SendMode sendMode = HttpEventCollectorSender.SendMode.Sequential, int batchInterval = HttpEventCollectorSender.DefaultBatchInterval, int batchSizeBytes = HttpEventCollectorSender.DefaultBatchSize, int batchSizeCount = HttpEventCollectorSender.DefaultBatchCount) : this(uri, token, metadata, sendMode, batchInterval, batchSizeBytes, batchSizeCount, (new HttpEventCollectorResendMiddleware(retriesOnError)).Plugin) { }
/// <summary> /// HttpEventCollectorTraceListener c-or. /// </summary> /// <param name="uri">Splunk server uri, for example https://localhost:8089.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="batchInterval">Batch interval in milliseconds.</param> /// <param name="batchSizeBytes">Batch max size.</param> /// <param name="batchSizeCount">MNax number of individual events in batch.</param> /// <param name="middleware"> /// HTTP client middleware. This allows to plug an HttpClient handler that /// intercepts logging HTTP traffic. /// </param> public HttpEventCollectorTraceListener( Uri uri, string token, HttpEventCollectorEventInfo.Metadata metadata = null, HttpEventCollectorSender.SendMode sendMode = HttpEventCollectorSender.SendMode.Sequential, int batchInterval = HttpEventCollectorSender.DefaultBatchInterval, int batchSizeBytes = HttpEventCollectorSender.DefaultBatchSize, int batchSizeCount = HttpEventCollectorSender.DefaultBatchCount, HttpEventCollectorSender.HttpEventCollectorMiddleware middleware = null) { sender = new HttpEventCollectorSender( uri, token, metadata, sendMode, batchInterval, batchSizeBytes, batchSizeCount, middleware); }
/// <summary> /// Serialize event info into a json string /// </summary> /// <param name="eventInfo"></param> /// <returns></returns> public static string SerializeEventInfo(HttpEventCollectorEventInfo eventInfo) { return JsonConvert.SerializeObject(eventInfo); }
/// <summary> /// Send an event to Splunk HTTP endpoint. Actual event send is done /// asynchronously and this method doesn't block client application. /// </summary> /// <param name="id">Event id.</param> /// <param name="severity">Event severity info.</param> /// <param name="message">Event message text.</param> /// <param name="data">Additional event data.</param> public void Send( string id = null, string severity = null, string message = null, object data = null) { HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(id, severity, message, data, metadata); // we use lock serializedEventsBatch to synchronize both // serializedEventsBatch and serializedEvents string serializedEventInfo = SerializeEventInfo(ei); lock (eventsBatchLock) { eventsBatch.Add(ei); serializedEventsBatch.Append(serializedEventInfo); if (eventsBatch.Count >= batchSizeCount || serializedEventsBatch.Length >= batchSizeBytes) { // there are enough events in the batch FlushInternal(); } } }