/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
/// <returns>Response from Elasticsearch</returns>
protected virtual ElasticsearchResponse <T> EmitBatchChecked <T>(IEnumerable <LogEvent> events) where T : class
{
// ReSharper disable PossibleMultipleEnumeration
if (events == null || !events.Any())
{
return(null);
}
var payload = new List <string>();
foreach (var e in events)
{
var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime());
var action = default(object);
if (string.IsNullOrWhiteSpace(_state.Options.PipelineName))
{
action = new { index = new { _index = indexName, _type = _state.Options.TypeName } };
}
else
{
action = new { index = new { _index = indexName, _type = _state.Options.TypeName, pipeline = _state.Options.PipelineName } };
}
var actionJson = _state.Serialize(action);
payload.Add(actionJson);
var sw = new StringWriter();
_state.Formatter.Format(e, sw);
payload.Add(sw.ToString());
}
return(_state.Client.Bulk <T>(payload));
}
/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
/// <returns>Response from Elasticsearch</returns>
protected virtual ElasticsearchResponse <T> EmitBatchChecked <T>(IEnumerable <LogEvent> events) where T : class
{
// ReSharper disable PossibleMultipleEnumeration
if (events == null || !events.Any())
{
return(null);
}
if (!_state.TemplateRegistrationSuccess && _state.Options.RegisterTemplateFailure == RegisterTemplateRecovery.FailSink)
{
return(null);
}
try
{
var payload = new List <string>();
foreach (var e in events)
{
var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime());
var action = default(object);
var pipelineName = _state.Options.PipelineNameDecider?.Invoke(e) ?? _state.Options.PipelineName;
if (string.IsNullOrWhiteSpace(pipelineName))
{
action = new { index = new { _index = indexName, _type = _state.Options.TypeName } };
}
else
{
action = new { index = new { _index = indexName, _type = _state.Options.TypeName, pipeline = pipelineName } };
}
var actionJson = _state.Serialize(action);
payload.Add(actionJson);
var sw = new StringWriter();
_state.Formatter.Format(e, sw);
payload.Add(sw.ToString());
}
return(_state.Client.Bulk <T>(payload));
}
catch (Exception ex)
{
HandleException(ex, events);
return(new ElasticsearchResponse <T>(ex));
}
}
/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
/// <remarks>
/// Override either <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatch(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" />
/// or <see cref="M:Serilog.Sinks.PeriodicBatching.PeriodicBatchingSink.EmitBatchAsync(System.Collections.Generic.IEnumerable{Serilog.Events.LogEvent})" />,
/// not both.
/// </remarks>
protected override void EmitBatch(IEnumerable <LogEvent> events)
{
// ReSharper disable PossibleMultipleEnumeration
if (events == null || !events.Any())
{
return;
}
var payload = new List <string>();
foreach (var e in events)
{
var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime());
var action = new { index = new { _index = indexName, _type = _state.Options.TypeName } };
var actionJson = _state.Serialize(action);
payload.Add(actionJson);
var sw = new StringWriter();
_state.Formatter.Format(e, sw);
payload.Add(sw.ToString());
}
_state.Client.Bulk(payload);
}
/// <summary>
/// Emit a batch of log events, running to completion synchronously.
/// </summary>
/// <param name="events">The events to emit.</param>
/// <returns>Response from Elasticsearch</returns>
protected virtual ElasticsearchResponse <DynamicDictionary> EmitBatchChecked(IEnumerable <LogEvent> events)
{
// ReSharper disable PossibleMultipleEnumeration
if (events == null || !events.Any())
{
return(null);
}
var payload = new List <string>();
foreach (var e in events)
{
var indexName = _state.GetIndexForEvent(e, e.Timestamp.ToUniversalTime());
var action = new { index = new { _index = indexName, _type = _state.Options.TypeName } };
var actionJson = _state.Serialize(action);
payload.Add(actionJson);
var sw = new StringWriter();
_state.Formatter.Format(e, sw);
payload.Add(sw.ToString());
}
return(_state.Client.Bulk(payload));
}
void OnTick()
{
try
{
// on the very first timer tick, we make the auto-register-if-necessary call
if (!_didRegisterTemplateIfNeeded)
{
_state.RegisterTemplateIfNeeded();
_didRegisterTemplateIfNeeded = true;
}
var count = 0;
do
{
// Locking the bookmark ensures that though there may be multiple instances of this
// class running, only one will ship logs at a time.
using (var bookmark = File.Open(_bookmarkFilename, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
{
long nextLineBeginsAtOffset;
string currentFilePath;
TryReadBookmark(bookmark, out nextLineBeginsAtOffset, out currentFilePath);
var fileSet = GetFileSet();
if (currentFilePath == null || !File.Exists(currentFilePath))
{
nextLineBeginsAtOffset = 0;
currentFilePath = fileSet.FirstOrDefault();
}
if (currentFilePath == null)
{
continue;
}
count = 0;
// file name pattern: whatever-bla-bla-20150218.json, whatever-bla-bla-20150218_1.json, etc.
var lastToken = currentFilePath.Split('-').Last();
// lastToken should be something like 20150218.json or 20150218_3.json now
if (!lastToken.ToLowerInvariant().EndsWith(".json"))
{
throw new FormatException(string.Format("The file name '{0}' does not seem to follow the right file pattern - it must be named [whatever]-{{Date}}[_n].json", Path.GetFileName(currentFilePath)));
}
var dateString = lastToken.Substring(0, 8);
var date = DateTime.ParseExact(dateString, "yyyyMMdd", CultureInfo.InvariantCulture);
var indexName = _state.GetIndexForEvent(null, date);
var payload = new List <string>();
using (var current = File.Open(currentFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
current.Position = nextLineBeginsAtOffset;
string nextLine;
while (count < _batchPostingLimit && TryReadLine(current, ref nextLineBeginsAtOffset, out nextLine))
{
var action = new { index = new { _index = indexName, _type = _state.Options.TypeName } };
var actionJson = _state.Serialize(action);
payload.Add(actionJson);
payload.Add(nextLine);
++count;
}
}
if (count > 0)
{
var response = _state.Client.Bulk <DynamicResponse>(payload);
if (response.Success)
{
WriteBookmark(bookmark, nextLineBeginsAtOffset, currentFilePath);
_connectionSchedule.MarkSuccess();
}
else
{
_connectionSchedule.MarkFailure();
SelfLog.WriteLine("Received failed ElasticSearch shipping result {0}: {1}", response.HttpStatusCode, response.OriginalException);
break;
}
}
else
{
// For whatever reason, there's nothing waiting to send. This means we should try connecting again at the
// regular interval, so mark the attempt as successful.
_connectionSchedule.MarkSuccess();
// Only advance the bookmark if no other process has the
// current file locked, and its length is as we found it.
if (fileSet.Length == 2 && fileSet.First() == currentFilePath && IsUnlockedAtLength(currentFilePath, nextLineBeginsAtOffset))
{
WriteBookmark(bookmark, 0, fileSet[1]);
}
if (fileSet.Length > 2)
{
// Once there's a third file waiting to ship, we do our
// best to move on, though a lock on the current file
// will delay this.
File.Delete(fileSet[0]);
}
}
}
}while (count == _batchPostingLimit);
}
catch (Exception ex)
{
SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex);
_connectionSchedule.MarkFailure();
}
finally
{
lock (_stateLock)
{
if (!_unloading)
{
SetTimer();
}
}
}
}