/// <summary>
/// Writes an array of logging events to the log target. By default it iterates on all
/// events and passes them to "Write" method. Inheriting classes can use this method to
/// optimize batch writes.
/// </summary>
/// <param name="logEvents">Logging events to be written out.</param>
protected async override void Write(IList <AsyncLogEventInfo> logEvents)
{
//must sort into containers and then into the blobs for the container
if (_getTableNameDelegate == null)
{
_getTableNameDelegate = c => TableName.Render(c.LogEvent);
}
var tableBuckets = SortHelpers.BucketSort(logEvents, _getTableNameDelegate);
//Iterate over all the tables being written to
foreach (var tableBucket in tableBuckets)
{
var tableNameFinal = CheckAndRepairTableNamingRules(tableBucket.Key);
InitializeTable(tableNameFinal);
var batch = new TableBatchOperation();
//add each message for the destination table limit batch to 100 elements
foreach (var asyncLogEventInfo in tableBucket.Value)
{
var entity = new NLogEntity(asyncLogEventInfo.LogEvent, Layout);
batch.Insert(entity);
if (batch.Count == 100)
{
await _table.ExecuteBatchAsync(batch);
batch.Clear();
}
}
if (batch.Count > 0)
{
await _table.ExecuteBatchAsync(batch);
}
}
}
/// <summary>
/// Writes an array of logging events to the log target. By default it iterates on all
/// events and passes them to "Write" method. Inheriting classes can use this method to
/// optimize batch writes.
/// </summary>
/// <param name="logEvents">Logging events to be written out.</param>
protected override Task WriteAsyncTask(IList<LogEventInfo> logEvents, CancellationToken cancellationToken)
{
//must sort into containers and then into the blobs for the container
if (_getContainerBlobNameDelegate == null)
_getContainerBlobNameDelegate = logEvent => new ContainerBlobKey(RenderLogEvent(Container, logEvent), RenderLogEvent(BlobName, logEvent));
if (logEvents.Count == 1)
{
return WriteToBlobAsync(logEvents, RenderLogEvent(Container, logEvents[0]), RenderLogEvent(BlobName, logEvents[0]), cancellationToken);
}
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getContainerBlobNameDelegate);
IList<Task> multipleTasks = partitionBuckets.Count > 1 ? new List<Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
try
{
var sendTask = WriteToBlobAsync(partitionBucket.Value, partitionBucket.Key.ContainerName, partitionBucket.Key.BlobName, cancellationToken);
if (multipleTasks == null)
return sendTask;
else
multipleTasks.Add(sendTask);
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureBlobStorage(Name={0}): Failed to write {1} logevents to blob. ContainerName={2}, BlobName={3}", Name, partitionBucket.Value.Count, partitionBucket.Key.ContainerName, partitionBucket.Key.BlobName);
if (multipleTasks == null)
throw;
}
}
return Task.WhenAll(multipleTasks ?? new Task[0]);
}
protected override Task WriteAsyncTask(IList <LogEventInfo> logEvents, CancellationToken cancellationToken)
{
if (_getEventHubPartitionKeyDelegate == null)
{
_getEventHubPartitionKeyDelegate = l => RenderLogEvent(PartitionKey, l);
}
if (logEvents.Count == 1)
{
var eventDataList = CreateEventDataList(logEvents, out var eventDataSize);
return(WriteSingleBatchAsync(eventDataList, _getEventHubPartitionKeyDelegate(logEvents[0])));
}
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getEventHubPartitionKeyDelegate);
IList <Task> multipleTasks = partitionBuckets.Count > 1 ? new List <Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
try
{
var eventDataList = CreateEventDataList(partitionBucket.Value, out var eventDataSize);
Task sendTask = Task.CompletedTask;
int batchSize = CalculateBatchSize(eventDataList, eventDataSize);
if (eventDataList.Count <= batchSize)
{
sendTask = WriteSingleBatchAsync(eventDataList, partitionBucket.Key);
}
else
{
// Must chain the tasks together so they don't run concurrently
foreach (var batchItem in GenerateBatches(eventDataList, batchSize))
{
string partitionKey = partitionBucket.Key;
sendTask = sendTask.ContinueWith(async p => await WriteSingleBatchAsync(batchItem, partitionKey).ConfigureAwait(false), cancellationToken);
}
}
if (multipleTasks == null)
{
return(sendTask);
}
else
{
multipleTasks.Add(sendTask);
}
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureEventHub(Name={0}): Failed to create EventData batch.", Name);
if (multipleTasks == null)
{
throw;
}
}
}
return(multipleTasks?.Count > 0 ? Task.WhenAll(multipleTasks) : Task.CompletedTask);
}
/// <summary>
/// Writes an array of logging events to the log target. By default it iterates on all
/// events and passes them to "Write" method. Inheriting classes can use this method to
/// optimize batch writes.
/// </summary>
/// <param name="logEvents">Logging events to be written out.</param>
protected override Task WriteAsyncTask(IList <LogEventInfo> logEvents, CancellationToken cancellationToken)
{
//must sort into containers and then into the blobs for the container
if (_getContainerBlobNameDelegate == null)
{
_getContainerBlobNameDelegate = logEvent => new ContainerBlobKey(RenderLogEvent(Container, logEvent), RenderLogEvent(BlobName, logEvent));
}
if (logEvents.Count == 1)
{
var containerName = RenderLogEvent(Container, logEvents[0]);
var blobName = RenderLogEvent(BlobName, logEvents[0]);
try
{
var blobPayload = CreateBlobPayload(logEvents);
return(WriteToBlobAsync(blobPayload, containerName, blobName, cancellationToken));
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureBlobStorage(Name={0}): Failed writing {1} logevents to BlobName={2} in ContainerName={3}", Name, 1, blobName, containerName);
throw;
}
}
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getContainerBlobNameDelegate);
IList <Task> multipleTasks = partitionBuckets.Count > 1 ? new List <Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
var containerName = partitionBucket.Key.ContainerName;
var blobName = partitionBucket.Key.BlobName;
var bucketSize = partitionBucket.Value.Count;
try
{
var blobPayload = CreateBlobPayload(partitionBucket.Value);
var sendTask = WriteToBlobAsync(blobPayload, containerName, blobName, cancellationToken);
if (multipleTasks == null)
{
return(sendTask);
}
multipleTasks.Add(sendTask);
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureBlobStorage(Name={0}): Failed writing {1} logevents to BlobName={2} in ContainerName={3}", Name, bucketSize, blobName, containerName);
if (multipleTasks == null)
{
throw;
}
}
}
return(Task.WhenAll(multipleTasks ?? new Task[0]));
}
// <inheritdoc />
protected override void Write(IList <AsyncLogEventInfo> logEvents)
{
if (_buildKeyStringDelegate == null)
{
_buildKeyStringDelegate = logEvent => RenderLogEvent(Key, logEvent.LogEvent);
}
var buckets = logEvents.BucketSort(_buildKeyStringDelegate);
foreach (var bucket in buckets)
{
WrappedTarget.WriteAsyncLogEvents(bucket.Value);
}
}
/// <summary>
/// Writes an array of logging events to the log target. By default it iterates on all
/// events and passes them to "Write" method. Inheriting classes can use this method to
/// optimize batch writes.
/// </summary>
/// <param name="logEvents">Logging events to be written out.</param>
protected override void Write(IList <AsyncLogEventInfo> logEvents)
{
//must sort into containers and then into the blobs for the container
if (_getContainerNameDelegate == null)
{
_getContainerNameDelegate = c => Container.Render(c.LogEvent);
}
var containerBuckets = SortHelpers.BucketSort(logEvents, _getContainerNameDelegate);
//Iterate over all the containers being written to
foreach (var containerBucket in containerBuckets)
{
var containerName = CheckAndRepairContainerNamingRules(containerBucket.Key);
InitializeContainer(containerName);
if (_getBlobNameDelegate == null)
{
_getBlobNameDelegate = c => BlobName.Render(c.LogEvent);
}
var blobBuckets = SortHelpers.BucketSort(containerBucket.Value, _getBlobNameDelegate);
//Iterate over all the blobs in the container to be written to
foreach (var blobBucket in blobBuckets)
{
var blobName = CheckAndRepairBlobNamingRules(blobBucket.Key);
InitializeBlob(blobName);
//Initilize StringBuilder size based on number of items to write. Default StringBuilder initialization size is 16 characters.
var logMessage = new StringBuilder(blobBucket.Value.Count * 128);
//add each message for the destination append blob
foreach (var asyncLogEventInfo in blobBucket.Value)
{
logMessage.AppendLine(Layout.Render(asyncLogEventInfo.LogEvent));
}
_appendBlob.AppendText(logMessage.ToString());
}
}
}
protected override Task WriteAsyncTask(IList <LogEventInfo> logEvents, CancellationToken cancellationToken)
{
if (_getEventHubPartitionKeyDelegate == null)
{
_getEventHubPartitionKeyDelegate = l => RenderLogEvent(PartitionKey, l);
}
if (logEvents.Count == 1)
{
var eventDataBatch = CreateEventDataBatch(logEvents, out var eventDataSize);
return(WriteSingleBatchAsync(eventDataBatch, _getEventHubPartitionKeyDelegate(logEvents[0])));
}
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getEventHubPartitionKeyDelegate);
IList <Task> multipleTasks = partitionBuckets.Count > 1 ? new List <Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
try
{
var eventDataBatch = CreateEventDataBatch(partitionBucket.Value, out var eventDataSize);
Task sendTask = WritePartitionBucketAsync(eventDataBatch, partitionBucket.Key, eventDataSize);
if (multipleTasks == null)
{
return(sendTask);
}
multipleTasks.Add(sendTask);
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureEventHub(Name={0}): Failed to create EventData batch.", Name);
if (multipleTasks == null)
{
throw;
}
}
}
return(multipleTasks?.Count > 0 ? Task.WhenAll(multipleTasks) : Task.CompletedTask);
}
protected override Task WriteAsyncTask(IList <LogEventInfo> logEvents, CancellationToken cancellationToken)
{
//must sort into containers and then into the blobs for the container
if (_getTablePartitionNameDelegate == null)
{
_getTablePartitionNameDelegate = logEvent => new TablePartitionKey(RenderLogEvent(TableName, logEvent), RenderLogEvent(PartitionKey, logEvent));
}
if (logEvents.Count == 1)
{
var batchItem = GenerateBatch(logEvents, RenderLogEvent(PartitionKey, logEvents[0]));
return(WriteToTableAsync(RenderLogEvent(TableName, logEvents[0]), batchItem, cancellationToken));
}
const int BatchMaxSize = 100;
var partitionBuckets = SortHelpers.BucketSort(logEvents, _getTablePartitionNameDelegate);
IList <Task> multipleTasks = partitionBuckets.Count > 1 ? new List <Task>(partitionBuckets.Count) : null;
foreach (var partitionBucket in partitionBuckets)
{
string tableName = partitionBucket.Key.TableName;
try
{
if (partitionBucket.Value.Count <= BatchMaxSize)
{
var batchItem = GenerateBatch(partitionBucket.Value, partitionBucket.Key.PartitionId);
var writeTask = WriteToTableAsync(partitionBucket.Key.TableName, batchItem, cancellationToken);
if (multipleTasks == null)
{
return(writeTask);
}
multipleTasks.Add(writeTask);
}
else
{
// Must chain the tasks together so they don't run concurrently
var batchCollection = GenerateBatches(partitionBucket.Value, partitionBucket.Key.PartitionId, BatchMaxSize);
Task writeTask = WriteMultipleBatchesAsync(batchCollection, tableName, cancellationToken);
if (multipleTasks == null)
{
return(writeTask);
}
multipleTasks.Add(writeTask);
}
}
catch (Exception ex)
{
InternalLogger.Error(ex, "AzureTableStorageTarget(Name={0}): Failed to write table={1}", Name, tableName);
if (multipleTasks == null)
{
throw;
}
}
}
return(Task.WhenAll(multipleTasks ?? new Task[0]));
}
