private SinkTrace TraceSource( RequestHandler handler, HttpEventCollectorEventInfo.Metadata metadata = null, HttpEventCollectorSender.SendMode sendMode = HttpEventCollectorSender.SendMode.Parallel, int batchInterval = 0, int batchSizeBytes = 0, int batchSizeCount = 0, HttpEventCollectorSender.HttpEventCollectorMiddleware middleware = null) { var listener = new ObservableEventListener(); var sink = new HttpEventCollectorSink( uri: uri, token: token, formatter: new TestEventFormatter(), metadata: metadata, sendMode: sendMode, batchInterval: batchInterval, batchSizeBytes: batchSizeBytes, batchSizeCount: batchSizeCount, middleware: MiddlewareInterceptor(handler, middleware)); listener.Subscribe(sink); var eventSource = TestEventSource.GetInstance(); listener.EnableEvents(eventSource, EventLevel.LogAlways, Keywords.All); return(new SinkTrace() { Source = eventSource, Sink = sink, Listener = listener }); }
static void VerifyErrorsAreRaised() { string indexName = "errorindex"; string tokenName = "errortoken"; double testStartTime = SplunkCliWrapper.GetEpochTime(); SplunkCliWrapper splunk = new SplunkCliWrapper(); string token = CreateIndexAndToken(splunk, tokenName, indexName); var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; var validMetaData = new HttpEventCollectorEventInfo.Metadata(index: indexName, source: "host", sourceType: "log", host: "customhostname"); var invalidMetaData = new HttpEventCollectorEventInfo.Metadata(index: "notexistingindex", source: "host", sourceType: "log", host: "customhostname"); var listenerWithWrongToken = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: "notexistingtoken", metadata: validMetaData); var listenerWithWrongUri = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8087"), token: token, metadata: validMetaData); var listenerWithWrongMetadata = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: token, metadata: invalidMetaData); bool wrongTokenWasRaised = false; listenerWithWrongToken.AddLoggingFailureHandler((HttpEventCollectorException e) => { wrongTokenWasRaised = true; }); bool wrongUriWasRaised = false; listenerWithWrongUri.AddLoggingFailureHandler((HttpEventCollectorException e) => { wrongUriWasRaised = true; }); bool wrongMetaDataWasRaised = false; listenerWithWrongMetadata.AddLoggingFailureHandler((HttpEventCollectorException e) => { wrongMetaDataWasRaised = true; }); trace.Listeners.Add(listenerWithWrongToken); trace.Listeners.Add(listenerWithWrongUri); trace.Listeners.Add(listenerWithWrongMetadata); // Generate data int eventCounter = GenerateData(trace); Console.WriteLine("{0} events were created, waiting for errors to be raised.", eventCounter); Thread.Sleep(30 * 1000); trace.Close(); Assert.True(wrongTokenWasRaised); Assert.True(wrongUriWasRaised); Assert.True(wrongMetaDataWasRaised); }
// Input trace listener private TraceSource Trace( RequestHandler handler, HttpEventCollectorEventInfo.Metadata metadata = null, int batchInterval = 0, int batchSizeBytes = 0, int batchSizeCount = 0, HttpEventCollectorSender.SendMode sendMode = HttpEventCollectorSender.SendMode.Parallel, HttpEventCollectorSender.HttpEventCollectorMiddleware middleware = null) { var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; trace.Listeners.Add( new HttpEventCollectorTraceListener( uri: uri, token: token, metadata: metadata, sendMode: sendMode, batchInterval: batchInterval, batchSizeBytes: batchSizeBytes, batchSizeCount: batchSizeCount, middleware: MiddlewareInterceptor(handler, middleware)) ); return(trace); }
/// <param name="uri">Splunk server uri, for example https://localhost:8088.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="batchInterval">Batch interval in milliseconds.</param> /// <param name="batchSizeBytes">Batch max size.</param> /// <param name="batchSizeCount">Max number of individual events in batch.</param> /// <param name="ignoreSslErrors">Server validation callback should always return true</param> /// <param name="middleware"> /// HTTP client middleware. This allows to plug an HttpClient handler that /// intercepts logging HTTP traffic. /// </param> /// <param name="formatter"></param> /// <remarks> /// Zero values for the batching params mean that batching is off. /// </remarks> public HttpEventCollectorSender( Uri uri, string token, HttpEventCollectorEventInfo.Metadata metadata, SendMode sendMode, int batchInterval, int batchSizeBytes, int batchSizeCount, bool ignoreSslErrors, HttpEventCollectorMiddleware middleware, HttpEventCollectorFormatter formatter = null) { this.httpEventCollectorEndpointUri = new Uri(uri, HttpEventCollectorPath); this.jsonSerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; this.jsonSerializerSettings.Formatting = Formatting.None; this.jsonSerializerSettings.Converters = new[] { new Newtonsoft.Json.Converters.StringEnumConverter() }; this.jsonSerializer = JsonSerializer.CreateDefault(this.jsonSerializerSettings); this.sendMode = sendMode; this.batchInterval = batchInterval; this.batchSizeBytes = batchSizeBytes; this.batchSizeCount = batchSizeCount; this.metadata = metadata; this.token = token; this.middleware = middleware; this.formatter = formatter; // special case - if batch interval is specified without size and count // they are set to "infinity", i.e., batch may have any size if (this.batchInterval > 0 && this.batchSizeBytes == 0 && this.batchSizeCount == 0) { this.batchSizeBytes = this.batchSizeCount = int.MaxValue; } // when size configuration setting is missing it's treated as "infinity", // i.e., any value is accepted. if (this.batchSizeCount == 0 && this.batchSizeBytes > 0) { this.batchSizeCount = int.MaxValue; } else if (this.batchSizeBytes == 0 && this.batchSizeCount > 0) { this.batchSizeBytes = int.MaxValue; } // setup the timer if (batchInterval != 0) // 0 means - no timer { timer = new Timer(OnTimer, null, batchInterval, batchInterval); } // setup HTTP client try { var httpMessageHandler = ignoreSslErrors ? BuildHttpMessageHandler(ignoreSslErrors) : null; httpClient = httpMessageHandler != null ? new HttpClient(httpMessageHandler) : new HttpClient(); } catch { // Fallback on PlatformNotSupported and other funny exceptions httpClient = new HttpClient(); } httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(AuthorizationHeaderScheme, token); }
public HttpEventCollectorSender( Uri uri, string token, HttpEventCollectorEventInfo.Metadata metadata, SendMode sendMode, int batchInterval, int batchSizeBytes, int batchSizeCount, HttpEventCollectorMiddleware middleware, HttpEventCollectorFormatter formatter = null, bool ignoreCertificateErrors = false ) { this.httpEventCollectorEndpointUri = new Uri(uri, HttpEventCollectorPath); this.sendMode = sendMode; this.batchSizeBytes = batchSizeBytes; this.batchSizeCount = batchSizeCount; this.metadata = metadata; this.token = token; this.middleware = middleware; this.formatter = formatter; if (batchInterval > 0 && this.batchSizeBytes == 0 && this.batchSizeCount == 0) { this.batchSizeBytes = this.batchSizeCount = int.MaxValue; } if (this.batchSizeCount == 0 && this.batchSizeBytes > 0) { this.batchSizeCount = int.MaxValue; } else if (this.batchSizeBytes == 0 && this.batchSizeCount > 0) { this.batchSizeBytes = int.MaxValue; } if (batchInterval != 0) { timer = new Timer(OnTimer, null, batchInterval, batchInterval); } if (ignoreCertificateErrors) { var certificateHandler = new HttpClientHandler { ClientCertificateOptions = ClientCertificateOption.Manual, ServerCertificateCustomValidationCallback = (httpRequestMessage, cert, cetChain, policyErrors) => true }; httpClient = new HttpClient(certificateHandler, true); } else { httpClient = new HttpClient(); } httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( AuthorizationHeaderScheme, token ); }
public HttpEventCollectorEventInfoBatch( Func <byte[], CancellationToken, Task <HttpStatusCode> > postEvents, JsonSerializerSettings jsonSerializerSettings, HttpEventCollectorEventInfo.Metadata metadata, HttpEventCollectorFormatter formatter = null) { this.postEvents = postEvents ?? throw new ArgumentNullException(nameof(postEvents)); this.jsonSerializerSettings = jsonSerializerSettings ?? throw new ArgumentNullException(nameof(jsonSerializerSettings)); jsonSerializer = JsonSerializer.CreateDefault(this.jsonSerializerSettings); this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); this.formatter = formatter; }
/// <summary> /// Initializes a new instance of the <see cref="HttpEventCollectorSender"/> class. /// </summary> /// <param name="uri">Splunk server uri, for example https://localhost:8088.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="channel">HTTP event collector data channel.</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="ignoreSslErrors">Server validation callback should always return true</param> /// <param name="proxyConfig">Default web proxy is used if set to true; otherwise, no proxy is used</param> /// <param name="maxConnectionsPerServer"></param> /// <param name="formatter">The formatter.</param> /// <param name="httpVersion10Hack">Fix for http version 1.0 servers</param> /// <remarks> /// Zero values for the batching params mean that batching is off. /// </remarks> public HttpEventCollectorSender( Uri uri, string token, string channel, HttpEventCollectorEventInfo.Metadata metadata, SendMode sendMode, bool ignoreSslErrors, ProxyConfiguration proxyConfig, int maxConnectionsPerServer, HttpEventCollectorFormatter formatter = null, bool httpVersion10Hack = false) { NLog.Common.InternalLogger.Debug("Initializing Splunk HttpEventCollectorSender"); httpEventCollectorEndpointUri = new Uri(uri, HttpEventCollectorPath); jsonSerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; jsonSerializerSettings.Formatting = Formatting.None; jsonSerializerSettings.Converters = new[] { new Newtonsoft.Json.Converters.StringEnumConverter() }; this.sendMode = sendMode; this.metadata = metadata; this.token = token; this.channel = channel; this.formatter = formatter; applyHttpVersion10Hack = httpVersion10Hack; // setup HTTP client try { var httpMessageHandler = BuildHttpMessageHandler(ignoreSslErrors, maxConnectionsPerServer, proxyConfig); httpClient = new HttpClient(httpMessageHandler); } catch { // Fallback on PlatformNotSupported and other funny exceptions httpClient = new HttpClient(); } // setup splunk header token httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(AuthorizationHeaderScheme, token); if (applyHttpVersion10Hack) { httpClient.BaseAddress = uri; httpClient.DefaultRequestHeaders.ConnectionClose = false; httpClient.DefaultRequestHeaders.Add("Connection", "keep-alive"); } // setup splunk channel request header if (!string.IsNullOrWhiteSpace(channel)) { httpClient.DefaultRequestHeaders.Add(ChannelRequestHeaderName, channel); } }
/// <summary> /// Send an event to Splunk HTTP endpoint. Actual event send is done /// asynchronously and this method doesn't block client application. /// </summary> /// <param name="id">Event id.</param> /// <param name="severity">Event severity info.</param> /// <param name="message">Event message text.</param> /// <param name="data">Additional event data.</param> /// <param name="metadataOverride">Metadata to use for this send.</param> public void Send( string id = null, string severity = null, string message = null, object data = null, HttpEventCollectorEventInfo.Metadata metadataOverride = null) { HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(id, severity, message, data, metadataOverride ?? metadata); DoSerialization(ei); }
public void HttpEventCollectorSinkCoreTest() { // authorization var trace = TraceSource((token, events) => { Assert.True(token == "TOKEN-GUID", "authentication"); return(new Response()); }); trace.Source.Message("", ""); trace.Dispose(); // metadata var metadata = new HttpEventCollectorEventInfo.Metadata( index: "main", source: "localhost", sourceType: "log", host: "demohost" ); trace = TraceSource( metadata: metadata, handler: (token, events) => { Assert.True(events[0].Event.Message == "EventId=1 EventName=MessageInfo Level=Error \"FormattedMessage=world - hello\" \"message=hello\" \"caller=world\"\r\n"); Assert.True(events[0].Index == "main"); Assert.True(events[0].Source == "localhost"); Assert.True(events[0].SourceType == "log"); Assert.True(events[0].Host == "demohost"); return(new Response()); } ); trace.Source.Message("hello", "world"); trace.Dispose(); // timestamp // metadata double now = (DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; trace = TraceSource( handler: (token, events) => { // check that timestamp is correct double time = double.Parse(events[0].Timestamp); Assert.True(time - now < 10.0); // it cannot be more than 10s after sending event return(new Response()); } ); trace.Source.Message("", ""); trace.Dispose(); }
/// <summary> /// Send an event to Splunk HTTP endpoint. Actual event send is done /// asynchronously and this method doesn't block client application. /// </summary> /// <param name="id">Event id.</param> /// <param name="level">Event severity info.</param> /// <param name="messageTemplate">Event message template.</param> /// <param name="renderedMessage">Event message rendered.</param> /// <param name="exception">Event exception.</param> /// <param name="properties">Additional event data.</param> /// <param name="metadataOverride">Metadata to use for this send.</param> public void Send( string id = null, string level = null, string messageTemplate = null, string renderedMessage = null, object exception = null, object properties = null, HttpEventCollectorEventInfo.Metadata metadataOverride = null) { HttpEventCollectorEventInfo ei = new HttpEventCollectorEventInfo(id, level, messageTemplate, renderedMessage, exception, properties, metadataOverride ?? metadata); DoSerialization(ei); }
/// <param name="uri">Splunk server uri, for example https://localhost:8088.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="batchInterval">Batch interval in milliseconds.</param> /// <param name="batchSizeBytes">Batch max size.</param> /// <param name="batchSizeCount">Max number of individual events in batch.</param> /// <param name="middleware"> /// HTTP client middleware. This allows to plug an HttpClient handler that /// intercepts logging HTTP traffic. /// </param> /// <remarks> /// Zero values for the batching params mean that batching is off. /// </remarks> public HttpEventCollectorSender( Uri uri, string token, HttpEventCollectorEventInfo.Metadata metadata, SendMode sendMode, int batchInterval, int batchSizeBytes, int batchSizeCount, HttpEventCollectorMiddleware middleware, HttpEventCollectorFormatter formatter = null) { this.serializer = new JsonSerializer(); serializer.NullValueHandling = NullValueHandling.Ignore; serializer.ContractResolver = new CamelCasePropertyNamesContractResolver(); this.httpEventCollectorEndpointUri = new Uri(uri, HttpEventCollectorPath); this.sendMode = sendMode; this.batchInterval = batchInterval; this.batchSizeBytes = batchSizeBytes; this.batchSizeCount = batchSizeCount; this.metadata = metadata; this.token = token; this.middleware = middleware; this.formatter = formatter; // special case - if batch interval is specified without size and count // they are set to "infinity", i.e., batch may have any size if (this.batchInterval > 0 && this.batchSizeBytes == 0 && this.batchSizeCount == 0) { this.batchSizeBytes = this.batchSizeCount = int.MaxValue; } // when size configuration setting is missing it's treated as "infinity", // i.e., any value is accepted. if (this.batchSizeCount == 0 && this.batchSizeBytes > 0) { this.batchSizeCount = int.MaxValue; } else if (this.batchSizeBytes == 0 && this.batchSizeCount > 0) { this.batchSizeBytes = int.MaxValue; } // setup the timer if (batchInterval != 0) // 0 means - no timer { timer = new Timer(OnTimer, null, batchInterval, batchInterval); } // setup HTTP client httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(AuthorizationHeaderScheme, token); }
/// <summary> /// HttpEventCollectorTraceListener c-or. Instantiates HttpEventCollectorTraceListener /// when retriesOnError parameter is specified. /// </summary> /// <param name="uri">Splunk server uri, for example https://localhost:8088.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="retriesOnError">Number of retries when network problem is detected</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="batchInterval">Batch interval in milliseconds.</param> /// <param name="batchSizeBytes">Batch max size.</param> /// <param name="batchSizeCount">MNax number of individual events in batch.</param> public HttpEventCollectorTraceListener( Uri uri, string token, int retriesOnError, HttpEventCollectorEventInfo.Metadata metadata = null, HttpEventCollectorSender.SendMode sendMode = HttpEventCollectorSender.SendMode.Sequential, int batchInterval = HttpEventCollectorSender.DefaultBatchInterval, int batchSizeBytes = HttpEventCollectorSender.DefaultBatchSize, int batchSizeCount = HttpEventCollectorSender.DefaultBatchCount) : this(uri, token, metadata, sendMode, batchInterval, batchSizeBytes, batchSizeCount, (new HttpEventCollectorResendMiddleware(retriesOnError)).Plugin) { }
static void VerifyEventsAreInOrder() { string tokenName = "verifyeventsareinordertoken"; string indexName = "verifyeventsareinorderindex"; SplunkCliWrapper splunk = new SplunkCliWrapper(); double testStartTime = SplunkCliWrapper.GetEpochTime(); string token = CreateIndexAndToken(splunk, tokenName, indexName); var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; var meta = new HttpEventCollectorEventInfo.Metadata(index: indexName, source: "host", sourceType: "log", host: "customhostname"); var listener = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: token, metadata: meta); trace.Listeners.Add(listener); // Generate data int totalEvents = 1000; string[] filer = new string[2]; filer[0] = new string('s', 1); filer[1] = new string('l', 100000); for (int i = 0; i < totalEvents; i++) { trace.TraceInformation(string.Format("TraceInformation. This is event {0}. {1}", i, filer[i % 2])); } string searchQuery = "index=" + indexName; Console.WriteLine("{0} events were created, waiting for indexing to complete.", totalEvents); splunk.WaitForIndexingToComplete(indexName); int eventsFound = splunk.GetSearchCount(searchQuery); Console.WriteLine("Indexing completed, {0} events were found. Elapsed time {1:F2} seconds", eventsFound, SplunkCliWrapper.GetEpochTime() - testStartTime); // Verify all events were indexed correctly and in order Assert.Equal(totalEvents, eventsFound); List <string> searchResults = splunk.GetSearchResults(searchQuery); Assert.Equal(searchResults.Count, eventsFound); for (int i = 0; i < totalEvents; i++) { int id = totalEvents - i - 1; string expected = string.Format("TraceInformation. This is event {0}", id); Assert.True(searchResults[i].Contains(expected)); } trace.Close(); }
/// <summary> /// HttpEventCollectorEventSink c-or with middleware parameter. /// </summary> /// <param name="uri">Splunk server uri, for example https://localhost:8088.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="formatter">Event formatter converting EventEntry instance into a string.</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="batchInterval">Batch interval in milliseconds.</param> /// <param name="batchSizeBytes">Batch max size.</param> /// <param name="batchSizeCount">MNax number of individual events in batch.</param> /// <param name="middleware"> /// HTTP client middleware. This allows to plug an HttpClient handler that /// intercepts logging HTTP traffic. /// </param> public HttpEventCollectorSink( Uri uri, string token, IEventTextFormatter formatter, HttpEventCollectorEventInfo.Metadata metadata = null, HttpEventCollectorSender.SendMode sendMode = HttpEventCollectorSender.SendMode.Sequential, int batchInterval = HttpEventCollectorSender.DefaultBatchInterval, int batchSizeBytes = HttpEventCollectorSender.DefaultBatchSize, int batchSizeCount = HttpEventCollectorSender.DefaultBatchCount, HttpEventCollectorSender.HttpEventCollectorMiddleware middleware = null) { this.formatter = formatter; sender = new HttpEventCollectorSender( uri, token, metadata, sendMode, batchInterval, batchSizeBytes, batchSizeCount, middleware); }
public static void SendEventsBatchedByTime() { string tokenName = "batchedbytimetoken"; string indexName = "batchedbytimeindex"; SplunkCliWrapper splunk = new SplunkCliWrapper(); double testStartTime = SplunkCliWrapper.GetEpochTime(); string token = CreateIndexAndToken(splunk, tokenName, indexName); var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; var meta = new HttpEventCollectorEventInfo.Metadata(index: indexName, source: "host", sourceType: "log", host: "customhostname"); var listener = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: token, metadata: meta, batchInterval: 1000); trace.Listeners.Add(listener); GenerateDataWaitForIndexingCompletion(splunk, indexName, testStartTime, trace); trace.Close(); }
static void SendEventsUnBatched() { string tokenName = "unbatchedtoken"; string indexName = "unbatchedindex"; SplunkCliWrapper splunk = new SplunkCliWrapper(); double testStartTime = SplunkCliWrapper.GetEpochTime(); string token = CreateIndexAndToken(splunk, tokenName, indexName); var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; var meta = new HttpEventCollectorEventInfo.Metadata(index: indexName, source: "host", sourceType: "log", host: "customhostname"); var listener = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: token, metadata: meta); trace.Listeners.Add(listener); GenerateDataWaitForIndexingCompletion(splunk, indexName, testStartTime, trace); trace.Close(); }
public void VerifyFlushEvents() { string tokenName = "flusheventtoken"; string indexName = "flusheventdindex"; SplunkCliWrapper splunk = new SplunkCliWrapper(); double testStartTime = SplunkCliWrapper.GetEpochTime(); string token = CreateIndexAndToken(splunk, tokenName, indexName); splunk.StopServer(); Thread.Sleep(5 * 1000); var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; var meta = new HttpEventCollectorEventInfo.Metadata(index: indexName, source: "host", sourceType: "log", host: "customhostname"); var listener = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: token, retriesOnError: int.MaxValue, metadata: meta); trace.Listeners.Add(listener); // Generate data, wait a little bit so retries are happenning and start Splunk. Expecting to see all data make it const int eventsToGeneratePerLoop = 250; const int expectedCount = eventsToGeneratePerLoop * 7; int eventCounter = GenerateData(trace, eventsPerLoop: eventsToGeneratePerLoop); splunk.StartServer(); trace.Close(); // Verify every event made to Splunk splunk.WaitForIndexingToComplete(indexName, stabilityPeriod: 30); int eventsFound = splunk.GetSearchCount("index=" + indexName); Assert.Equal(expectedCount, eventsFound); }
static void VerifyErrorsAreRaised() { string indexName = "errorindex"; string tokenName = "errortoken"; double testStartTime = SplunkCliWrapper.GetEpochTime(); SplunkCliWrapper splunk = new SplunkCliWrapper(); string token = CreateIndexAndToken(splunk, tokenName, indexName); var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; var validMetaData = new HttpEventCollectorEventInfo.Metadata(index: indexName, source: "host", sourceType: "log", host: "customhostname"); var invalidMetaData = new HttpEventCollectorEventInfo.Metadata(index: "notexistingindex", source: "host", sourceType: "log", host: "customhostname"); var listenerWithWrongToken = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: "notexistingtoken", metadata: validMetaData); var listenerWithWrongUri = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8087"), token: token, metadata: validMetaData); var listenerWithWrongMetadata = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: token, metadata: invalidMetaData); bool wrongTokenWasRaised = false; listenerWithWrongToken.AddLoggingFailureHandler((HttpEventCollectorException e) => { wrongTokenWasRaised = true; }); bool wrongUriWasRaised = false; listenerWithWrongUri.AddLoggingFailureHandler((HttpEventCollectorException e) => { wrongUriWasRaised = true; }); bool wrongMetaDataWasRaised = false; listenerWithWrongMetadata.AddLoggingFailureHandler((HttpEventCollectorException e) => { wrongMetaDataWasRaised = true; }); trace.Listeners.Add(listenerWithWrongToken); trace.Listeners.Add(listenerWithWrongUri); trace.Listeners.Add(listenerWithWrongMetadata); // Generate data int eventCounter = GenerateData(trace); Console.WriteLine("{0} events were created, waiting for errors to be raised.", eventCounter); Thread.Sleep(30 * 1000); trace.Close(); Assert.True(wrongTokenWasRaised); Assert.True(wrongUriWasRaised); Assert.True(wrongMetaDataWasRaised); }
public void HttpEventCollectorSinkCoreTest() { // authorization var trace = TraceSource((token, events) => { Assert.True(token == "TOKEN-GUID", "authentication"); return new Response(); }); trace.Source.Message("", ""); trace.Dispose(); // metadata var metadata = new HttpEventCollectorEventInfo.Metadata( index: "main", source: "localhost", sourceType: "log", host: "demohost" ); trace = TraceSource( metadata: metadata, handler: (token, events) => { Assert.True(events[0].Event.Message == "EventId=1 EventName=MessageInfo Level=Error \"FormattedMessage=world - hello\" \"message=hello\" \"caller=world\"\r\n"); Assert.True(events[0].Index == "main"); Assert.True(events[0].Source == "localhost"); Assert.True(events[0].SourceType == "log"); Assert.True(events[0].Host == "demohost"); return new Response(); } ); trace.Source.Message("hello", "world"); trace.Dispose(); // timestamp // metadata double now = (DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; trace = TraceSource( handler: (token, events) => { // check that timestamp is correct double time = double.Parse(events[0].Timestamp); Assert.True(time - now < 10.0); // it cannot be more than 10s after sending event return new Response(); } ); trace.Source.Message("", ""); trace.Dispose(); }
public void HttpEventCollectorCoreTest() { // authorization var trace = Trace((token, input) => { Assert.True(token == "TOKEN-GUID", "authentication"); return(new Response()); }); trace.TraceInformation("info"); trace.Close(); // metadata double now = (DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; var metadata = new HttpEventCollectorEventInfo.Metadata( index: "main", source: "localhost", sourceType: "log", host: "demohost" ); trace = Trace( metadata: metadata, handler: (token, events) => { Assert.True(events[0].Index == "main"); Assert.True(events[0].Source == "localhost"); Assert.True(events[0].SourceType == "log"); Assert.True(events[0].Host == "demohost"); // check that timestamp is correct double time = double.Parse(events[0].Timestamp); Assert.True(time - now < 10.0); // it cannot be more than 10s after sending event return(new Response()); } ); trace.TraceInformation("info"); trace.Close(); // test various tracing commands trace = Trace((token, events) => { Assert.True(events[0].Event.Message == "info"); return(new Response()); }); trace.TraceInformation("info"); trace.Close(); trace = Trace((token, events) => { Assert.True(events[0].Event.Severity == "Information"); Assert.True(events[0].Event.Id == "1"); Assert.True(((string[])(events[0].Event.Data))[0] == "one"); Assert.True(((string[])(events[0].Event.Data))[1] == "two"); return(new Response()); }); trace.TraceData(TraceEventType.Information, 1, new string[] { "one", "two" }); trace.Close(); trace = Trace((token, events) => { Assert.True(events[0].Event.Severity == "Critical"); Assert.True(events[0].Event.Id == "2"); return(new Response()); }); trace.TraceEvent(TraceEventType.Critical, 2); trace.Close(); trace = Trace((token, events) => { Assert.True(events[0].Event.Severity == "Error"); Assert.True(events[0].Event.Id == "3"); Assert.True(events[0].Event.Message == "hello"); return(new Response()); }); trace.TraceEvent(TraceEventType.Error, 3, "hello"); trace.Close(); trace = Trace((token, events) => { Assert.True(events[0].Event.Severity == "Resume"); Assert.True(events[0].Event.Id == "4"); Assert.True(events[0].Event.Message == "hello world"); return(new Response()); }); trace.TraceEvent(TraceEventType.Resume, 4, "hello {0}", "world"); trace.Close(); Guid guid = new Guid("11111111-2222-3333-4444-555555555555"); trace = Trace((token, events) => { Assert.True(events[0].Event.Id == "5"); Assert.True(((Guid)(events[0].Event.Data)).CompareTo(guid) == 0); return(new Response()); }); trace.TraceTransfer(5, "transfer", guid); trace.Close(); }
static void VerifyEventsAreInOrder() { string tokenName = "verifyeventsareinordertoken"; string indexName = "verifyeventsareinorderindex"; SplunkCliWrapper splunk = new SplunkCliWrapper(); double testStartTime = SplunkCliWrapper.GetEpochTime(); string token = CreateIndexAndToken(splunk, tokenName, indexName); var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; var meta = new HttpEventCollectorEventInfo.Metadata(index: indexName, source: "host", sourceType: "log", host: "customhostname"); var listener = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: token, metadata: meta); trace.Listeners.Add(listener); // Generate data int totalEvents = 1000; string[] filer = new string[2]; filer[0] = new string('s', 1); filer[1] = new string('l', 100000); for (int i = 0; i < totalEvents; i++) { trace.TraceInformation(string.Format("TraceInformation. This is event {0}. {1}", i, filer[i%2])); } string searchQuery = "index=" + indexName; Console.WriteLine("{0} events were created, waiting for indexing to complete.", totalEvents); splunk.WaitForIndexingToComplete(indexName); int eventsFound = splunk.GetSearchCount(searchQuery); Console.WriteLine("Indexing completed, {0} events were found. Elapsed time {1:F2} seconds", eventsFound, SplunkCliWrapper.GetEpochTime() - testStartTime); // Verify all events were indexed correctly and in order Assert.Equal(totalEvents, eventsFound); List<string> searchResults = splunk.GetSearchResults(searchQuery); Assert.Equal(searchResults.Count, eventsFound); for (int i = 0; i< totalEvents; i++) { int id = totalEvents - i - 1; string expected = string.Format("TraceInformation. This is event {0}. {1}", id, filer[id % 2]); Assert.True(searchResults[i].Contains(expected)); } trace.Close(); }
public void HttpEventCollectorCoreTest() { // authorization var trace = Trace((token, input) => { Assert.True(token == "TOKEN-GUID", "authentication"); return new Response(); }); trace.TraceInformation("info"); trace.Close(); // metadata double now = (DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; var metadata = new HttpEventCollectorEventInfo.Metadata( index: "main", source: "localhost", sourceType: "log", host: "demohost" ); trace = Trace( metadata: metadata, handler: (token, events) => { Assert.True(events[0].Index == "main"); Assert.True(events[0].Source == "localhost"); Assert.True(events[0].SourceType == "log"); Assert.True(events[0].Host == "demohost"); // check that timestamp is correct double time = double.Parse(events[0].Timestamp); Assert.True(time - now < 10.0); // it cannot be more than 10s after sending event return new Response(); } ); trace.TraceInformation("info"); trace.Close(); // test various tracing commands trace = Trace((token, events) => { Assert.True(events[0].Event.Message == "info"); return new Response(); }); trace.TraceInformation("info"); trace.Close(); trace = Trace((token, events) => { Assert.True(events[0].Event.Severity == "Information"); Assert.True(events[0].Event.Id == "1"); Assert.True(((string[])(events[0].Event.Data))[0] == "one"); Assert.True(((string[])(events[0].Event.Data))[1] == "two"); return new Response(); }); trace.TraceData(TraceEventType.Information, 1, new string[] { "one", "two" }); trace.Close(); trace = Trace((token, events) => { Assert.True(events[0].Event.Severity == "Critical"); Assert.True(events[0].Event.Id == "2"); return new Response(); }); trace.TraceEvent(TraceEventType.Critical, 2); trace.Close(); trace = Trace((token, events) => { Assert.True(events[0].Event.Severity == "Error"); Assert.True(events[0].Event.Id == "3"); Assert.True(events[0].Event.Message == "hello"); return new Response(); }); trace.TraceEvent(TraceEventType.Error, 3, "hello"); trace.Close(); trace = Trace((token, events) => { Assert.True(events[0].Event.Severity == "Resume"); Assert.True(events[0].Event.Id == "4"); Assert.True(events[0].Event.Message == "hello world"); return new Response(); }); trace.TraceEvent(TraceEventType.Resume, 4, "hello {0}", "world"); trace.Close(); Guid guid = new Guid("11111111-2222-3333-4444-555555555555"); trace = Trace((token, events) => { Assert.True(events[0].Event.Id == "5"); Assert.True(((Guid)(events[0].Event.Data)).CompareTo(guid) == 0); return new Response(); }); trace.TraceTransfer(5, "transfer", guid); trace.Close(); }
/// <summary> /// Initializes a new instance of the <see cref="HttpEventCollectorSender"/> class. /// </summary> /// <param name="uri">Splunk server uri, for example https://localhost:8088.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="channel">HTTP event collector data channel.</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="batchInterval">Batch interval in milliseconds.</param> /// <param name="batchSizeBytes">Batch max size.</param> /// <param name="batchSizeCount">Max number of individual events in batch.</param> /// <param name="ignoreSslErrors">Server validation callback should always return true</param> /// <param name="useProxy">Default web proxy is used if set to true; otherwise, no proxy is used</param> /// <param name="middleware">HTTP client middleware. This allows to plug an HttpClient handler that /// intercepts logging HTTP traffic.</param> /// <param name="formatter">The formatter.</param> /// <remarks> /// Zero values for the batching params mean that batching is off. /// </remarks> public HttpEventCollectorSender( Uri uri, string token, string channel, HttpEventCollectorEventInfo.Metadata metadata, SendMode sendMode, int batchInterval, int batchSizeBytes, int batchSizeCount, bool ignoreSslErrors, bool useProxy, int maxConnectionsPerServer, HttpEventCollectorMiddleware middleware, HttpEventCollectorFormatter formatter = null, bool httpVersion10Hack = false) { NLog.Common.InternalLogger.Debug("Initializing Splunk HttpEventCollectorSender"); this.httpEventCollectorEndpointUri = new Uri(uri, HttpEventCollectorPath); this.jsonSerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; this.jsonSerializerSettings.Formatting = Formatting.None; this.jsonSerializerSettings.Converters = new[] { new Newtonsoft.Json.Converters.StringEnumConverter() }; this.jsonSerializer = JsonSerializer.CreateDefault(this.jsonSerializerSettings); this.sendMode = sendMode; this.batchInterval = batchInterval; this.batchSizeBytes = batchSizeBytes; this.batchSizeCount = batchSizeCount; this.metadata = metadata; this.token = token; this.channel = channel; this.middleware = middleware; this.formatter = formatter; this.applyHttpVersion10Hack = httpVersion10Hack; // special case - if batch interval is specified without size and count // they are set to "infinity", i.e., batch may have any size if (this.batchInterval > 0 && this.batchSizeBytes == 0 && this.batchSizeCount == 0) { this.batchSizeBytes = this.batchSizeCount = int.MaxValue; } // when size configuration setting is missing it's treated as "infinity", // i.e., any value is accepted. if (this.batchSizeCount == 0 && this.batchSizeBytes > 0) { this.batchSizeCount = int.MaxValue; } else if (this.batchSizeBytes == 0 && this.batchSizeCount > 0) { this.batchSizeBytes = int.MaxValue; } // setup the timer if (batchInterval != 0) // 0 means - no timer { timer = new Timer(OnTimer, null, batchInterval, batchInterval); } // setup HTTP client try { var httpMessageHandler = BuildHttpMessageHandler(ignoreSslErrors, useProxy, maxConnectionsPerServer); httpClient = new HttpClient(httpMessageHandler); } catch { // Fallback on PlatformNotSupported and other funny exceptions httpClient = new HttpClient(); } // Enable TLS 1.2 NLog.Common.InternalLogger.Debug("Setting TLS1.2 SecurityProtocolType"); ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; // setup splunk header token httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(AuthorizationHeaderScheme, token); if (this.applyHttpVersion10Hack) { httpClient.BaseAddress = uri; httpClient.DefaultRequestHeaders.ConnectionClose = false; httpClient.DefaultRequestHeaders.Add("Connection", "keep-alive"); } // setup splunk channel request header if (!string.IsNullOrWhiteSpace(channel)) { httpClient.DefaultRequestHeaders.Add(ChannelRequestHeaderName, channel); } }
/// <param name="uri">Splunk server uri, for example https://localhost:8089.</param> /// <param name="token">HTTP event collector authorization token.</param> /// <param name="metadata">Logger metadata.</param> /// <param name="sendMode">Send mode of the events.</param> /// <param name="batchInterval">Batch interval in milliseconds.</param> /// <param name="batchSizeBytes">Batch max size.</param> /// <param name="batchSizeCount">MNax number of individual events in batch.</param> /// <param name="middleware"> /// HTTP client middleware. This allows to plug an HttpClient handler that /// intercepts logging HTTP traffic. /// </param> /// <remarks> /// Zero values for the batching params mean that batching is off. /// </remarks> public HttpEventCollectorSender( Uri uri, string token, HttpEventCollectorEventInfo.Metadata metadata, SendMode sendMode, int batchInterval, int batchSizeBytes, int batchSizeCount, HttpEventCollectorMiddleware middleware) { this.httpEventCollectorEndpointUri = new Uri(uri, HttpEventCollectorPath); this.sendMode = sendMode; this.batchInterval = batchInterval; this.batchSizeBytes = batchSizeBytes; this.batchSizeCount = batchSizeCount; this.metadata = metadata; this.token = token; this.middleware = middleware; // special case - if batch interval is specified without size and count // they are set to "infinity", i.e., batch may have any size if (this.batchInterval > 0 && this.batchSizeBytes == 0 && this.batchSizeCount == 0) { this.batchSizeBytes = this.batchSizeCount = int.MaxValue; } // when size configuration setting is missing it's treated as "infinity", // i.e., any value is accepted. if (this.batchSizeCount == 0 && this.batchSizeBytes > 0) { this.batchSizeCount = int.MaxValue; } else if (this.batchSizeBytes == 0 && this.batchSizeCount > 0) { this.batchSizeBytes = int.MaxValue; } // setup the timer if (batchInterval != 0) // 0 means - no timer { timer = new Timer(OnTimer, null, batchInterval, batchInterval); } // setup HTTP client httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(AuthorizationHeaderScheme, token); }
public void HttpEventCollectorSenderMetadataOverrideTest() { Func <String, List <HttpEventCollectorEventInfo>, Response> noopHandler = (token, events) => { return(new Response()); }; HttpEventCollectorEventInfo.Metadata defaultmetadata = new HttpEventCollectorEventInfo.Metadata(index: "defaulttestindex", source: "defaulttestsource", sourceType: "defaulttestsourcetype", host: "defaulttesthost"); HttpEventCollectorEventInfo.Metadata overridemetadata = new HttpEventCollectorEventInfo.Metadata(index: "overridetestindex", source: "overridetestsource", sourceType: "overridetestsourcetype", host: "overridetesthost"); HttpEventCollectorSender httpEventCollectorSender = new HttpEventCollectorSender(uri, "TOKEN-GUID", defaultmetadata, HttpEventCollectorSender.SendMode.Sequential, 100000, 100000, 3, new HttpEventCollectorSender.HttpEventCollectorMiddleware((token, events, next) => { Assert.True(events.Count == 3); // Id = id1 should have the default meta data values. Assert.True(events[0].Event.Id == "id1"); Assert.True(events[0].Index == defaultmetadata.Index); Assert.True(events[0].Source == defaultmetadata.Source); Assert.True(events[0].SourceType == defaultmetadata.SourceType); Assert.True(events[0].Host == defaultmetadata.Host); // Id = id2 should have the metadataOverride values. Assert.True(events[1].Event.Id == "id2"); Assert.True(events[1].Index == overridemetadata.Index); Assert.True(events[1].Source == overridemetadata.Source); Assert.True(events[1].SourceType == overridemetadata.SourceType); Assert.True(events[1].Host == overridemetadata.Host); // Id = id3 should have the default meta data values. Assert.True(events[2].Event.Id == "id3"); Assert.True(events[2].Index == defaultmetadata.Index); Assert.True(events[2].Source == defaultmetadata.Source); Assert.True(events[2].SourceType == defaultmetadata.SourceType); Assert.True(events[2].Host == defaultmetadata.Host); Response response = noopHandler(token, events); HttpResponseMessage httpResponseMessage = new HttpResponseMessage(); httpResponseMessage.StatusCode = response.Code; byte[] buf = Encoding.UTF8.GetBytes(response.Context); httpResponseMessage.Content = new StringContent(response.Context); var task = new Task <HttpResponseMessage>(() => { return(httpResponseMessage); }); task.RunSynchronously(); return(task); }) ); httpEventCollectorSender.Send(id: "id1"); httpEventCollectorSender.Send(id: "id2", metadataOverride: overridemetadata); httpEventCollectorSender.Send(id: "id3"); httpEventCollectorSender.FlushSync(); httpEventCollectorSender.Dispose(); }
public void VerifyFlushEvents() { string tokenName = "flusheventtoken"; string indexName = "flusheventdindex"; SplunkCliWrapper splunk = new SplunkCliWrapper(); double testStartTime = SplunkCliWrapper.GetEpochTime(); string token = CreateIndexAndToken(splunk, tokenName, indexName); splunk.StopServer(); Thread.Sleep(5 * 1000); var trace = new TraceSource("HttpEventCollectorLogger"); trace.Switch.Level = SourceLevels.All; var meta = new HttpEventCollectorEventInfo.Metadata(index: indexName, source: "host", sourceType: "log", host: "customhostname"); var listener = new HttpEventCollectorTraceListener( uri: new Uri("https://127.0.0.1:8088"), token: token, retriesOnError: int.MaxValue, metadata: meta); trace.Listeners.Add(listener); // Generate data, wait a little bit so retries are happenning and start Splunk. Expecting to see all data make it const int eventsToGeneratePerLoop = 250; const int expectedCount = eventsToGeneratePerLoop * 7; int eventCounter = GenerateData(trace, eventsPerLoop: eventsToGeneratePerLoop); splunk.StartServer(); trace.Close(); // Verify every event made to Splunk splunk.WaitForIndexingToComplete(indexName, stabilityPeriod: 30); int eventsFound = splunk.GetSearchCount("index=" + indexName); Assert.Equal(expectedCount, eventsFound); }