public async Task SuccessAsync(Replicatable replicatable, TimeSpan replicatedIn = default, CancellationToken cancellationToken = default)
{
var table = await CreateOrGetCloudTableAsync("Succeeded", cancellationToken);
var operation = TableOperation.InsertOrMerge(CreateLogEntry(replicatable, replicatedIn.TotalSeconds));
await table.ExecuteAsync(operation, cancellationToken);
}
public async Task FailureAsync(Replicatable replicatable, Exception e, CancellationToken cancellationToken = default)
{
var table = await CreateOrGetCloudTableAsync("Failed", cancellationToken);
var operation = TableOperation.InsertOrMerge(CreateLogEntry(replicatable, e));
await table.ExecuteAsync(operation, cancellationToken);
}
public async Task SendAsync(Replicatable replicatable, string queueName, CancellationToken cancellationToken = default)
{
var queueClient = GetOrCreateQueueClientIfNotExists(queueName, cancellationToken);
var serializedMessage = JsonConvert.SerializeObject(replicatable);
await queueClient.SendMessageAsync(serializedMessage, timeToLive : TimeSpan.FromSeconds(-1), cancellationToken : cancellationToken);
}
public async Task PurgeFailureAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
{
var table = await CreateOrGetCloudTableAsync("Failed", cancellationToken);
var operation = TableOperation.Delete(CreateLogEntry(replicatable));
await table.ExecuteAsync(operation, cancellationToken);
}
public async Task <ILogEntry> GetFailedAsync(Replicatable replicatable, CancellationToken cancellationToken)
{
var table = await CreateOrGetCloudTableAsync("Failed", cancellationToken);
var operation = TableOperation.Retrieve <LogEntry>(replicatable.PartitionKey(), replicatable.RowKey());
return((await table.ExecuteAsync(operation, cancellationToken)).Result as ILogEntry);
}
public async Task TryPurgeFailureAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
{
try
{
await PurgeFailureAsync(replicatable, cancellationToken);
}
catch { }
}
private LogEntry CreateLogEntry(Replicatable replicatable)
{
return(new LogEntry
{
PartitionKey = replicatable.PartitionKey(),
RowKey = replicatable.RowKey(),
ETag = "*"
});
}
public async Task FailureAsync(Replicatable replicatable, Exception exception, CancellationToken cancellationToken = default)
{
var exceptionTelemetry = new ExceptionTelemetry(exception);
AddDiagnosticInfo(exceptionTelemetry.Properties, replicatable.DiagnosticInfo);
AddRequestId(exceptionTelemetry.Properties, exception?.RequestId());
_telemetryClient.TrackException(exceptionTelemetry);
await Task.CompletedTask;
}
private LogEntry CreateLogEntry(Replicatable replicatable, double?replicatedInSeconds = null, Exception exception = null)
{
return(new LogEntry
{
PartitionKey = replicatable.PartitionKey(),
RowKey = replicatable.RowKey(),
Source = replicatable.Source,
Destination = replicatable.Destination,
Content = JsonConvert.SerializeObject(replicatable),
ReplicatedInSeconds = replicatedInSeconds,
ReplicatedBytes = exception == null ? replicatable.SizeInBytes : null,
ExceptionReason = exception?.Message,
Exception = exception?.ToString(),
CorrelationId = _correlationId,
RequestId = exception?.RequestId()
});
}
public async Task ReplicateAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
{
if (!replicatable.SizeInBytes.HasValue)
{
replicatable.SizeInBytes = await GetContentSizeInBytesAsync(replicatable.Source, cancellationToken);
}
if (!await ExistsWithCopySuccessAndSimilarSizeAsync(replicatable.Destination, replicatable.SizeInBytes))
{
await EnsureContainerExistsAsync(replicatable.Destination, cancellationToken);
if (CanBeProcessedInASingleRequest(replicatable))
{
await CopyBlobFromUrlAsync(replicatable, cancellationToken);
}
else
{
await CopyBlocksFromUrlAsync(replicatable, cancellationToken);
}
}
}
private async Task CopyBlocksFromUrlAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
{
var blockSize = _replicationConfiguration.ReplicationBlockSizeInBytes ?? ReplicationConfigurationDefaults.ReplicationBlockSizeInBytes;
var numberOfBlocks = Math.Ceiling((decimal)replicatable.SizeInBytes / blockSize);
var blockIds = new List <string>();
var tasks = new List <Task>();
for (var blockCounter = 0; blockCounter < numberOfBlocks; blockCounter++)
{
var blockId = _blobStorage.GenerateBlockId(blockSize * blockCounter);
var startByte = blockCounter * blockSize;
var endByte = Math.Min(startByte + blockSize - 1, replicatable.SizeInBytes.Value - 1);
tasks.Add(_blobStorage.PutBlockFromUrlAsync(replicatable.Source, replicatable.Destination, blockId, startByte, endByte, cancellationToken));
blockIds.Add(blockId);
}
await Task.WhenAll(tasks);
await _blobStorage.PutBlockListAsync(replicatable.Destination, blockIds, cancellationToken);
}
public async Task SuccessAsync(Replicatable replicatable, TimeSpan replicatedIn = default, CancellationToken cancellationToken = default)
{
var queuedTelemetry = new QueuedTelemetry
{
Name = _telemetryConfiguration.QueuedTelemetryName ?? TelemetryConfigurationDefaults.QueuedTelemetryName,
Duration = replicatedIn
};
queuedTelemetry.Metrics[_telemetryConfiguration.QueuedTelemetryMetricAggregatedCount ?? TelemetryConfigurationDefaults.QueuedTelemetryMetricAggregatedCount] = 1;
queuedTelemetry.Metrics[_telemetryConfiguration.QueuedTelemetryMetricAggregatedDuration ?? TelemetryConfigurationDefaults.QueuedTelemetryMetricAggregatedDuration] = replicatedIn.TotalSeconds;
if (replicatable.SizeInBytes.HasValue)
{
queuedTelemetry.Metrics[_telemetryConfiguration.QueuedTelemetryMetricAggregatedSize ?? TelemetryConfigurationDefaults.QueuedTelemetryMetricAggregatedSize] = (double)replicatable.SizeInBytes;
}
AddDiagnosticInfo(queuedTelemetry.Properties, replicatable.DiagnosticInfo);
_telemetryQueue.Queue(queuedTelemetry);
await Task.CompletedTask;
}
private async Task HandleMessageAsync(QueueMessage message)
{
Replicatable replicatable = null;
try
{
replicatable = MessageSerializer.Deserialize(message.MessageText);
using (var executionTimer = new ExecutionTimer())
{
await _messageHandler.HandleAsync(replicatable, _cancellationToken);
await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, _cancellationToken);
await Task.WhenAll(_replicationStatusLoggers.Select(r => r.SuccessAsync(replicatable, executionTimer.CalculateElapsedAndStopMeasuring())));
}
}
catch (UnsupportedMessageFormatException e)
{
await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, _cancellationToken);
await Task.WhenAll(_replicationStatusLoggers.Select(r => r.UnprocessableAsync(message.MessageId, message.PopReceipt, message.MessageText, e)));
}
catch (Exception e)
{
if (message.DequeueCount > _maximumNumberOfRetries)
{
await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, _cancellationToken);
await Task.WhenAll(_replicationStatusLoggers.Select(r => r.FailureAsync(replicatable, e)));
}
}
finally
{
_concurrencyLimiter.Release();
}
}
private bool CanBeProcessedInASingleRequest(Replicatable replicatable)
{
return((replicatable.SizeInBytes ?? 0) / 1024 / 1024 <= 255);
}
private async Task CopyBlobFromUrlAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
{
await _blobStorage.CopyBlobFromUrlAsync(replicatable.Source, replicatable.Destination, cancellationToken);
}
public async Task FailureAsync(Replicatable replicatable, Exception e, CancellationToken cancellationToken = default)
{
await _messageSender.SendAsync(replicatable, FailedQueueName, cancellationToken);
}
public async Task SuccessAsync(Replicatable replicatable, TimeSpan replicatedIn = default, CancellationToken cancellationToken = default)
{
await _messageSender.SendAsync(replicatable, SuccessQueueName, cancellationToken);
}
public static string RowKey(this Replicatable replicatable)
{
return(Hash($"{replicatable.Source}-{replicatable.Destination}"));
}
public static string Serialize(Replicatable replicatable)
{
return(JsonConvert.SerializeObject(replicatable));
}
private LogEntry CreateLogEntry(Replicatable replicatable, Exception e)
{
return(CreateLogEntry(replicatable, null, e));
}
public static string PartitionKey(this Replicatable replicatable)
{
return(Hash(ExtractHostFromUrl(replicatable.Source)));
}
public async Task HandleAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
{
await _blobReplicator.ReplicateAsync(replicatable, cancellationToken);
}