/// <summary>
/// Writes an array of logging events to the log target. By default it iterates on all
/// events and passes them to "Write" method. Inheriting classes can use this method to
/// optimize batch writes.
/// </summary>
/// <param name="logEvents">Logging events to be written out.</param>
protected override Task WriteAsyncTask(IList<LogEventInfo> logEvents, CancellationToken cancellationToken)
{
//must sort into containers and then into the blobs for the container
if (_getContainerBlobNameDelegate == null)
_getContainerBlobNameDelegate = logEvent => new ContainerBlobKey(RenderLogEvent(Container, logEvent), RenderLogEvent(BlobName, logEvent));
if (logEvents.Count == 1)
{
return WriteToBlobAsync(logEvents, RenderLogEvent(Container, logEvents[0]), RenderLogEvent(BlobName, logEvents[0]), cancellationToken);
}
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getContainerBlobNameDelegate);
IList<Task> multipleTasks = partitionBuckets.Count > 1 ? new List<Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
try
{
var sendTask = WriteToBlobAsync(partitionBucket.Value, partitionBucket.Key.ContainerName, partitionBucket.Key.BlobName, cancellationToken);
if (multipleTasks == null)
return sendTask;
else
multipleTasks.Add(sendTask);
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureBlobStorage(Name={0}): Failed to write {1} logevents to blob. ContainerName={2}, BlobName={3}", Name, partitionBucket.Value.Count, partitionBucket.Key.ContainerName, partitionBucket.Key.BlobName);
if (multipleTasks == null)
throw;
}
}
return Task.WhenAll(multipleTasks ?? new Task[0]);
}
protected override Task WriteAsyncTask(IList <LogEventInfo> logEvents, CancellationToken cancellationToken)
{
if (_getEventHubPartitionKeyDelegate == null)
{
_getEventHubPartitionKeyDelegate = l => RenderLogEvent(PartitionKey, l);
}
if (logEvents.Count == 1)
{
var eventDataList = CreateEventDataList(logEvents, out var eventDataSize);
return(WriteSingleBatchAsync(eventDataList, _getEventHubPartitionKeyDelegate(logEvents[0])));
}
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getEventHubPartitionKeyDelegate);
IList <Task> multipleTasks = partitionBuckets.Count > 1 ? new List <Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
try
{
var eventDataList = CreateEventDataList(partitionBucket.Value, out var eventDataSize);
Task sendTask = Task.CompletedTask;
int batchSize = CalculateBatchSize(eventDataList, eventDataSize);
if (eventDataList.Count <= batchSize)
{
sendTask = WriteSingleBatchAsync(eventDataList, partitionBucket.Key);
}
else
{
// Must chain the tasks together so they don't run concurrently
foreach (var batchItem in GenerateBatches(eventDataList, batchSize))
{
string partitionKey = partitionBucket.Key;
sendTask = sendTask.ContinueWith(async p => await WriteSingleBatchAsync(batchItem, partitionKey).ConfigureAwait(false), cancellationToken);
}
}
if (multipleTasks == null)
{
return(sendTask);
}
else
{
multipleTasks.Add(sendTask);
}
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureEventHub(Name={0}): Failed to create EventData batch.", Name);
if (multipleTasks == null)
{
throw;
}
}
}
return(multipleTasks?.Count > 0 ? Task.WhenAll(multipleTasks) : Task.CompletedTask);
}
/// <summary>
/// Writes an array of logging events to the log target. By default it iterates on all
/// events and passes them to "Write" method. Inheriting classes can use this method to
/// optimize batch writes.
/// </summary>
/// <param name="logEvents">Logging events to be written out.</param>
protected override Task WriteAsyncTask(IList <LogEventInfo> logEvents, CancellationToken cancellationToken)
{
//must sort into containers and then into the blobs for the container
if (_getContainerBlobNameDelegate == null)
{
_getContainerBlobNameDelegate = logEvent => new ContainerBlobKey(RenderLogEvent(Container, logEvent), RenderLogEvent(BlobName, logEvent));
}
if (logEvents.Count == 1)
{
var containerName = RenderLogEvent(Container, logEvents[0]);
var blobName = RenderLogEvent(BlobName, logEvents[0]);
try
{
var blobPayload = CreateBlobPayload(logEvents);
return(WriteToBlobAsync(blobPayload, containerName, blobName, cancellationToken));
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureBlobStorage(Name={0}): Failed writing {1} logevents to BlobName={2} in ContainerName={3}", Name, 1, blobName, containerName);
throw;
}
}
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getContainerBlobNameDelegate);
IList <Task> multipleTasks = partitionBuckets.Count > 1 ? new List <Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
var containerName = partitionBucket.Key.ContainerName;
var blobName = partitionBucket.Key.BlobName;
var bucketSize = partitionBucket.Value.Count;
try
{
var blobPayload = CreateBlobPayload(partitionBucket.Value);
var sendTask = WriteToBlobAsync(blobPayload, containerName, blobName, cancellationToken);
if (multipleTasks == null)
{
return(sendTask);
}
multipleTasks.Add(sendTask);
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureBlobStorage(Name={0}): Failed writing {1} logevents to BlobName={2} in ContainerName={3}", Name, bucketSize, blobName, containerName);
if (multipleTasks == null)
{
throw;
}
}
}
return(Task.WhenAll(multipleTasks ?? new Task[0]));
}
/// <summary>
/// Writes an array of logging events to the log target. By default it iterates on all
/// events and passes them to "Write" method. Inheriting classes can use this method to
/// optimize batch writes.
/// </summary>
/// <param name="logEvents">Logging events to be written out.</param>
protected override void Write(AsyncLogEventInfo[] logEvents)
{
var buckets = SortHelpers.BucketSort(logEvents, c => this.BuildConnectionString(c.LogEvent));
try
{
foreach (var kvp in buckets)
{
foreach (AsyncLogEventInfo ev in kvp.Value)
{
try
{
this.WriteEventToDatabase(ev.LogEvent);
ev.Continuation(null);
}
catch (Exception exception)
{
// in case of exception, close the connection and report it
InternalLogger.Error(exception, "Error when writing to database.");
if (exception.MustBeRethrownImmediately())
{
throw;
}
InternalLogger.Trace("DatabaseTarget: close connection because of exception");
this.CloseConnection();
ev.Continuation(exception);
if (exception.MustBeRethrown())
{
throw;
}
}
}
}
}
finally
{
if (!this.KeepConnection)
{
InternalLogger.Trace("DatabaseTarget: close connection because of KeepConnection=false");
this.CloseConnection();
}
}
}
protected override Task WriteAsyncTask(IList <LogEventInfo> logEvents, CancellationToken cancellationToken)
{
if (_getEventHubPartitionKeyDelegate == null)
{
_getEventHubPartitionKeyDelegate = l => RenderLogEvent(PartitionKey, l);
}
if (logEvents.Count == 1)
{
var eventDataBatch = CreateEventDataBatch(logEvents, out var eventDataSize);
return(WriteSingleBatchAsync(eventDataBatch, _getEventHubPartitionKeyDelegate(logEvents[0])));
}
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getEventHubPartitionKeyDelegate);
IList <Task> multipleTasks = partitionBuckets.Count > 1 ? new List <Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
try
{
var eventDataBatch = CreateEventDataBatch(partitionBucket.Value, out var eventDataSize);
Task sendTask = WritePartitionBucketAsync(eventDataBatch, partitionBucket.Key, eventDataSize);
if (multipleTasks == null)
{
return(sendTask);
}
multipleTasks.Add(sendTask);
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureEventHub(Name={0}): Failed to create EventData batch.", Name);
if (multipleTasks == null)
{
throw;
}
}
}
return(multipleTasks?.Count > 0 ? Task.WhenAll(multipleTasks) : Task.CompletedTask);
}
protected override Task WriteAsyncTask(IList <LogEventInfo> logEvents, CancellationToken cancellationToken)
{
//must sort into containers and then into the blobs for the container
if (_getTablePartitionNameDelegate == null)
{
_getTablePartitionNameDelegate = logEvent => new TablePartitionKey(RenderLogEvent(TableName, logEvent), RenderLogEvent(PartitionKey, logEvent));
}
if (logEvents.Count == 1)
{
var batchItem = GenerateBatch(logEvents, RenderLogEvent(PartitionKey, logEvents[0]));
return(WriteToTableAsync(RenderLogEvent(TableName, logEvents[0]), batchItem, cancellationToken));
}
const int BatchMaxSize = 100;
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getTablePartitionNameDelegate);
IList <Task> multipleTasks = partitionBuckets.Count > 1 ? new List <Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
string tableName = partitionBucket.Key.TableName;
try
{
if (partitionBucket.Value.Count <= BatchMaxSize)
{
var batchItem = GenerateBatch(partitionBucket.Value, partitionBucket.Key.PartitionId);
var writeTask = WriteToTableAsync(partitionBucket.Key.TableName, batchItem, cancellationToken);
if (multipleTasks == null)
{
return(writeTask);
}
multipleTasks.Add(writeTask);
}
else
{
// Must chain the tasks together so they don't run concurrently
var batchCollection = GenerateBatches(partitionBucket.Value, partitionBucket.Key.PartitionId, BatchMaxSize);
Task writeTask = WriteMultipleBatchesAsync(batchCollection, tableName, cancellationToken);
if (multipleTasks == null)
{
return(writeTask);
}
multipleTasks.Add(writeTask);
}
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureTableStorageTarget(Name={0}): Failed to write table={1}", Name, tableName);
if (multipleTasks == null)
{
throw;
}
}
}
return(Task.WhenAll(multipleTasks ?? new Task[0]));
}