/// <summary> /// Emit a batch of log events, running to completion synchronously. /// </summary> /// <param name="events">The events to emit.</param> /// <returns>Response from Elasticsearch</returns> protected virtual ElasticsearchResponse <T> EmitBatchChecked <T>(IEnumerable <LogEvent> events) where T : class { // ReSharper disable PossibleMultipleEnumeration if (events == null || !events.Any()) { return(null); } var payload = new List <string>(); foreach (var e in events) { var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime()); var action = default(object); if (string.IsNullOrWhiteSpace(_state.Options.PipelineName)) { action = new { index = new { _index = indexName, _type = _state.Options.TypeName } }; } else { action = new { index = new { _index = indexName, _type = _state.Options.TypeName, pipeline = _state.Options.PipelineName } }; } var actionJson = _state.Serialize(action); payload.Add(actionJson); var sw = new StringWriter(); _state.Formatter.Format(e, sw); payload.Add(sw.ToString()); } return(_state.Client.Bulk <T>(payload)); }
/// <summary> /// Emit a batch of log events, running to completion synchronously. /// </summary> /// <param name="events">The events to emit.</param> /// <returns>Response from Elasticsearch</returns> protected virtual ElasticsearchResponse <T> EmitBatchChecked <T>(IEnumerable <LogEvent> events) where T : class { // ReSharper disable PossibleMultipleEnumeration if (events == null || !events.Any()) { return(null); } if (!_state.TemplateRegistrationSuccess && _state.Options.RegisterTemplateFailure == RegisterTemplateRecovery.FailSink) { return(null); } try { var payload = new List <string>(); foreach (var e in events) { var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime()); var action = default(object); var pipelineName = _state.Options.PipelineNameDecider?.Invoke(e) ?? _state.Options.PipelineName; if (string.IsNullOrWhiteSpace(pipelineName)) { action = new { index = new { _index = indexName, _type = _state.Options.TypeName } }; } else { action = new { index = new { _index = indexName, _type = _state.Options.TypeName, pipeline = pipelineName } }; } var actionJson = _state.Serialize(action); payload.Add(actionJson); var sw = new StringWriter(); _state.Formatter.Format(e, sw); payload.Add(sw.ToString()); } return(_state.Client.Bulk <T>(payload)); } catch (Exception ex) { HandleException(ex, events); return(new ElasticsearchResponse <T>(ex)); } }
/// <summary> /// Emit a batch of log events, running to completion synchronously. /// </summary> /// <param name="events">The events to emit.</param> /// <remarks> /// Override either <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatch(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" /> /// or <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" />, /// not both. /// </remarks> protected override void EmitBatch(IEnumerable <LogEvent> events) { // ReSharper disable PossibleMultipleEnumeration if (events == null || !events.Any()) { return; } var payload = new List <string>(); foreach (var e in events) { var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime()); var action = new { index = new { _index = indexName, _type = _state.Options.TypeName } }; var actionJson = _state.Serialize(action); payload.Add(actionJson); var sw = new StringWriter(); _state.Formatter.Format(e, sw); payload.Add(sw.ToString()); } _state.Client.Bulk(payload); }
/// <summary> /// Emit a batch of log events, running to completion synchronously. /// </summary> /// <param name="events">The events to emit.</param> /// <returns>Response from Elasticsearch</returns> protected virtual ElasticsearchResponse <DynamicDictionary> EmitBatchChecked(IEnumerable <LogEvent> events) { // ReSharper disable PossibleMultipleEnumeration if (events == null || !events.Any()) { return(null); } var payload = new List <string>(); foreach (var e in events) { var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime()); var action = new { index = new { _index = indexName, _type = _state.Options.TypeName } }; var actionJson = _state.Serialize(action); payload.Add(actionJson); var sw = new StringWriter(); _state.Formatter.Format(e, sw); payload.Add(sw.ToString()); } return(_state.Client.Bulk(payload)); }
void OnTick() { try { // on the very first timer tick, we make the auto-register-if-necessary call if (!_didRegisterTemplateIfNeeded) { _state.RegisterTemplateIfNeeded(); _didRegisterTemplateIfNeeded = true; } var count = 0; do { // Locking the bookmark ensures that though there may be multiple instances of this // class running, only one will ship logs at a time. using (var bookmark = File.Open(_bookmarkFilename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read)) { long nextLineBeginsAtOffset; string currentFilePath; TryReadBookmark(bookmark, out nextLineBeginsAtOffset, out currentFilePath); var fileSet = GetFileSet(); if (currentFilePath == null || !File.Exists(currentFilePath)) { nextLineBeginsAtOffset = 0; currentFilePath = fileSet.FirstOrDefault(); } if (currentFilePath == null) { continue; } count = 0; // file name pattern: whatever-bla-bla-20150218.json, whatever-bla-bla-20150218_1.json, etc. var lastToken = currentFilePath.Split('-').Last(); // lastToken should be something like 20150218.json or 20150218_3.json now if (!lastToken.ToLowerInvariant().EndsWith(".json")) { throw new FormatException(string.Format("The file name '{0}' does not seem to follow the right file pattern - it must be named [whatever]-{{Date}}[_n].json", Path.GetFileName(currentFilePath))); } var dateString = lastToken.Substring(0, 8); var date = DateTime.ParseExact(dateString, "yyyyMMdd", CultureInfo.InvariantCulture); var indexName = _state.GetIndexForEvent(null, date); var payload = new List <string>(); using (var current = File.Open(currentFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { current.Position = nextLineBeginsAtOffset; string nextLine; while (count < _batchPostingLimit && TryReadLine(current, ref nextLineBeginsAtOffset, out nextLine)) { var action = new { index = new { _index = indexName, _type = _state.Options.TypeName } }; var actionJson = _state.Serialize(action); payload.Add(actionJson); payload.Add(nextLine); ++count; } } if (count > 0) { var response = _state.Client.Bulk <DynamicResponse>(payload); if (response.Success) { WriteBookmark(bookmark, nextLineBeginsAtOffset, currentFilePath); _connectionSchedule.MarkSuccess(); } else { _connectionSchedule.MarkFailure(); SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}", response.HttpStatusCode, response.OriginalException); break; } } else { // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the // regular interval, so mark the attempt as successful. _connectionSchedule.MarkSuccess(); // Only advance the bookmark if no other process has the // current file locked, and its length is as we found it. if (fileSet.Length == 2 && fileSet.First() == currentFilePath && IsUnlockedAtLength(currentFilePath, nextLineBeginsAtOffset)) { WriteBookmark(bookmark, 0, fileSet[1]); } if (fileSet.Length > 2) { // Once there's a third file waiting to ship, we do our // best to move on, though a lock on the current file // will delay this. File.Delete(fileSet[0]); } } } }while (count == _batchPostingLimit); } catch (Exception ex) { SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex); _connectionSchedule.MarkFailure(); } finally { lock (_stateLock) { if (!_unloading) { SetTimer(); } } } }
/// <summary> /// Emit a batch of log events, running to completion synchronously. /// </summary> /// <param name="events">The events to emit.</param> /// <remarks> /// Override either <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatch(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" /> /// or <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" />, /// not both. /// </remarks> protected override void EmitBatch(IEnumerable <LogEvent> events) { DynamicResponse result; try { result = this.EmitBatchChecked <DynamicResponse>(events); } catch (Exception ex) { HandleException(ex, events); return; } // Handle the results from ES, check if there are any errors. if (result.Success && result.Body?["errors"] == true) { var indexer = 0; var items = result.Body["items"]; foreach (var item in items) { if (item["index"] != null && HasProperty(item["index"], "error") && item["index"]["error"] != null) { var e = events.ElementAt(indexer); if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToSelfLog)) { // ES reports an error, output the error to the selflog SelfLog.WriteLine( "Failed to store event with template '{0}' into Elasticsearch. Elasticsearch reports for index {1} the following: {2}", e.MessageTemplate, item["index"]["_index"], _state.Serialize(item["index"]["error"])); } if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToFailureSink) && _state.Options.FailureSink != null) { // Send to a failure sink try { _state.Options.FailureSink.Emit(e); } catch (Exception ex) { // We do not let this fail too SelfLog.WriteLine("Caught exception while emitting to sink {1}: {0}", ex, _state.Options.FailureSink); } } if (_state.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.RaiseCallback) && _state.Options.FailureCallback != null) { // Send to a failure callback try { _state.Options.FailureCallback(e); } catch (Exception ex) { // We do not let this fail too SelfLog.WriteLine("Caught exception while emitting to callback {1}: {0}", ex, _state.Options.FailureCallback); } } } indexer++; } } else if (result.Success == false && result.OriginalException != null) { HandleException(result.OriginalException, events); } }