Ejemplo n.º 1
0
        public async Task SuccessAsync(Replicatable replicatable, TimeSpan replicatedIn = default, CancellationToken cancellationToken = default)
        {
            var table = await CreateOrGetCloudTableAsync("Succeeded", cancellationToken);

            var operation = TableOperation.InsertOrMerge(CreateLogEntry(replicatable, replicatedIn.TotalSeconds));
            await table.ExecuteAsync(operation, cancellationToken);
        }
Ejemplo n.º 2
0
        public async Task FailureAsync(Replicatable replicatable, Exception e, CancellationToken cancellationToken = default)
        {
            var table = await CreateOrGetCloudTableAsync("Failed", cancellationToken);

            var operation = TableOperation.InsertOrMerge(CreateLogEntry(replicatable, e));
            await table.ExecuteAsync(operation, cancellationToken);
        }
Ejemplo n.º 3
0
        public async Task SendAsync(Replicatable replicatable, string queueName, CancellationToken cancellationToken = default)
        {
            var queueClient = GetOrCreateQueueClientIfNotExists(queueName, cancellationToken);

            var serializedMessage = JsonConvert.SerializeObject(replicatable);
            await queueClient.SendMessageAsync(serializedMessage, timeToLive : TimeSpan.FromSeconds(-1), cancellationToken : cancellationToken);
        }
Ejemplo n.º 4
0
        public async Task PurgeFailureAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
        {
            var table = await CreateOrGetCloudTableAsync("Failed", cancellationToken);

            var operation = TableOperation.Delete(CreateLogEntry(replicatable));
            await table.ExecuteAsync(operation, cancellationToken);
        }
Ejemplo n.º 5
0
        public async Task <ILogEntry> GetFailedAsync(Replicatable replicatable, CancellationToken cancellationToken)
        {
            var table = await CreateOrGetCloudTableAsync("Failed", cancellationToken);

            var operation = TableOperation.Retrieve <LogEntry>(replicatable.PartitionKey(), replicatable.RowKey());

            return((await table.ExecuteAsync(operation, cancellationToken)).Result as ILogEntry);
        }
Ejemplo n.º 6
0
 public async Task TryPurgeFailureAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
 {
     try
     {
         await PurgeFailureAsync(replicatable, cancellationToken);
     }
     catch { }
 }
Ejemplo n.º 7
0
 private LogEntry CreateLogEntry(Replicatable replicatable)
 {
     return(new LogEntry
     {
         PartitionKey = replicatable.PartitionKey(),
         RowKey = replicatable.RowKey(),
         ETag = "*"
     });
 }
Ejemplo n.º 8
0
        public async Task FailureAsync(Replicatable replicatable, Exception exception, CancellationToken cancellationToken = default)
        {
            var exceptionTelemetry = new ExceptionTelemetry(exception);

            AddDiagnosticInfo(exceptionTelemetry.Properties, replicatable.DiagnosticInfo);
            AddRequestId(exceptionTelemetry.Properties, exception?.RequestId());

            _telemetryClient.TrackException(exceptionTelemetry);

            await Task.CompletedTask;
        }
Ejemplo n.º 9
0
 private LogEntry CreateLogEntry(Replicatable replicatable, double?replicatedInSeconds = null, Exception exception = null)
 {
     return(new LogEntry
     {
         PartitionKey = replicatable.PartitionKey(),
         RowKey = replicatable.RowKey(),
         Source = replicatable.Source,
         Destination = replicatable.Destination,
         Content = JsonConvert.SerializeObject(replicatable),
         ReplicatedInSeconds = replicatedInSeconds,
         ReplicatedBytes = exception == null ? replicatable.SizeInBytes : null,
         ExceptionReason = exception?.Message,
         Exception = exception?.ToString(),
         CorrelationId = _correlationId,
         RequestId = exception?.RequestId()
     });
 }
Ejemplo n.º 10
0
        public async Task ReplicateAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
        {
            if (!replicatable.SizeInBytes.HasValue)
            {
                replicatable.SizeInBytes = await GetContentSizeInBytesAsync(replicatable.Source, cancellationToken);
            }

            if (!await ExistsWithCopySuccessAndSimilarSizeAsync(replicatable.Destination, replicatable.SizeInBytes))
            {
                await EnsureContainerExistsAsync(replicatable.Destination, cancellationToken);

                if (CanBeProcessedInASingleRequest(replicatable))
                {
                    await CopyBlobFromUrlAsync(replicatable, cancellationToken);
                }
                else
                {
                    await CopyBlocksFromUrlAsync(replicatable, cancellationToken);
                }
            }
        }
Ejemplo n.º 11
0
        private async Task CopyBlocksFromUrlAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
        {
            var blockSize      = _replicationConfiguration.ReplicationBlockSizeInBytes ?? ReplicationConfigurationDefaults.ReplicationBlockSizeInBytes;
            var numberOfBlocks = Math.Ceiling((decimal)replicatable.SizeInBytes / blockSize);

            var blockIds = new List <string>();
            var tasks    = new List <Task>();

            for (var blockCounter = 0; blockCounter < numberOfBlocks; blockCounter++)
            {
                var blockId   = _blobStorage.GenerateBlockId(blockSize * blockCounter);
                var startByte = blockCounter * blockSize;
                var endByte   = Math.Min(startByte + blockSize - 1, replicatable.SizeInBytes.Value - 1);

                tasks.Add(_blobStorage.PutBlockFromUrlAsync(replicatable.Source, replicatable.Destination, blockId, startByte, endByte, cancellationToken));

                blockIds.Add(blockId);
            }

            await Task.WhenAll(tasks);

            await _blobStorage.PutBlockListAsync(replicatable.Destination, blockIds, cancellationToken);
        }
Ejemplo n.º 12
0
        public async Task SuccessAsync(Replicatable replicatable, TimeSpan replicatedIn = default, CancellationToken cancellationToken = default)
        {
            var queuedTelemetry = new QueuedTelemetry
            {
                Name     = _telemetryConfiguration.QueuedTelemetryName ?? TelemetryConfigurationDefaults.QueuedTelemetryName,
                Duration = replicatedIn
            };

            queuedTelemetry.Metrics[_telemetryConfiguration.QueuedTelemetryMetricAggregatedCount ?? TelemetryConfigurationDefaults.QueuedTelemetryMetricAggregatedCount] = 1;

            queuedTelemetry.Metrics[_telemetryConfiguration.QueuedTelemetryMetricAggregatedDuration ?? TelemetryConfigurationDefaults.QueuedTelemetryMetricAggregatedDuration] = replicatedIn.TotalSeconds;

            if (replicatable.SizeInBytes.HasValue)
            {
                queuedTelemetry.Metrics[_telemetryConfiguration.QueuedTelemetryMetricAggregatedSize ?? TelemetryConfigurationDefaults.QueuedTelemetryMetricAggregatedSize] = (double)replicatable.SizeInBytes;
            }

            AddDiagnosticInfo(queuedTelemetry.Properties, replicatable.DiagnosticInfo);

            _telemetryQueue.Queue(queuedTelemetry);

            await Task.CompletedTask;
        }
Ejemplo n.º 13
0
        private async Task HandleMessageAsync(QueueMessage message)
        {
            Replicatable replicatable = null;

            try
            {
                replicatable = MessageSerializer.Deserialize(message.MessageText);

                using (var executionTimer = new ExecutionTimer())
                {
                    await _messageHandler.HandleAsync(replicatable, _cancellationToken);

                    await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, _cancellationToken);

                    await Task.WhenAll(_replicationStatusLoggers.Select(r => r.SuccessAsync(replicatable, executionTimer.CalculateElapsedAndStopMeasuring())));
                }
            }
            catch (UnsupportedMessageFormatException e)
            {
                await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, _cancellationToken);

                await Task.WhenAll(_replicationStatusLoggers.Select(r => r.UnprocessableAsync(message.MessageId, message.PopReceipt, message.MessageText, e)));
            }
            catch (Exception e)
            {
                if (message.DequeueCount > _maximumNumberOfRetries)
                {
                    await _queueClient.DeleteMessageAsync(message.MessageId, message.PopReceipt, _cancellationToken);

                    await Task.WhenAll(_replicationStatusLoggers.Select(r => r.FailureAsync(replicatable, e)));
                }
            }
            finally
            {
                _concurrencyLimiter.Release();
            }
        }
Ejemplo n.º 14
0
 private bool CanBeProcessedInASingleRequest(Replicatable replicatable)
 {
     return((replicatable.SizeInBytes ?? 0) / 1024 / 1024 <= 255);
 }
Ejemplo n.º 15
0
 private async Task CopyBlobFromUrlAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
 {
     await _blobStorage.CopyBlobFromUrlAsync(replicatable.Source, replicatable.Destination, cancellationToken);
 }
Ejemplo n.º 16
0
 public async Task FailureAsync(Replicatable replicatable, Exception e, CancellationToken cancellationToken = default)
 {
     await _messageSender.SendAsync(replicatable, FailedQueueName, cancellationToken);
 }
Ejemplo n.º 17
0
 public async Task SuccessAsync(Replicatable replicatable, TimeSpan replicatedIn = default, CancellationToken cancellationToken = default)
 {
     await _messageSender.SendAsync(replicatable, SuccessQueueName, cancellationToken);
 }
Ejemplo n.º 18
0
 public static string RowKey(this Replicatable replicatable)
 {
     return(Hash($"{replicatable.Source}-{replicatable.Destination}"));
 }
Ejemplo n.º 19
0
 public static string Serialize(Replicatable replicatable)
 {
     return(JsonConvert.SerializeObject(replicatable));
 }
Ejemplo n.º 20
0
 private LogEntry CreateLogEntry(Replicatable replicatable, Exception e)
 {
     return(CreateLogEntry(replicatable, null, e));
 }
Ejemplo n.º 21
0
 public static string PartitionKey(this Replicatable replicatable)
 {
     return(Hash(ExtractHostFromUrl(replicatable.Source)));
 }
Ejemplo n.º 22
0
 public async Task HandleAsync(Replicatable replicatable, CancellationToken cancellationToken = default)
 {
     await _blobReplicator.ReplicateAsync(replicatable, cancellationToken);
 }