private async Task ProcessBlobAsync(Uri blobUri, CancellationToken token)
{
_logger.LogInformation("ProcessBlobAsync: Start to process blob to {BlobName}.", blobUri.AbsoluteUri);
if (token.IsCancellationRequested)
{
_logger.LogInformation("ProcessBlobAsync: The operation was cancelled.");
return;
}
using (var lockResult = await _source.TakeLockAsync(blobUri, token))
{
if (lockResult.LockIsTaken /*lockResult*/)
{
var operationToken = lockResult.BlobOperationToken.Token;
if (!operationToken.IsCancellationRequested)
{
using (var inputStream = await _source.OpenReadAsync(blobUri, ContentType.GZip, token))
{
var writeOperationResult = await _destination.TryWriteAsync(inputStream, ProcessStream, blobUri.Segments.Last(), ContentType.GZip, operationToken);
await _source.TryCleanAsync(lockResult, onError : writeOperationResult.OperationException != null, token : operationToken);
await _source.TryReleaseLockAsync(lockResult, operationToken);
}
}
}
}
_logger.LogInformation("ProcessBlobAsync: Finished to process blob {BlobName}", blobUri.AbsoluteUri);
}
/// <summary>
/// Try to process the files from the source.
/// After processing the file is cleaned. This means it wil be moved either to a archive or a deadletter container.
/// </summary>
/// <param name="maxFileCount">Only max this number of files will be processed at once.</param>
/// <param name="fileNameTransform">A Func to be used to generate the output file name fro the input filename.</param>
/// <param name="sourceContentType">The <see cref="Stats.AzureCdnLogs.Common.Collect.ContentType" for the source file./></param>
/// <param name="destinationContentType">The <see cref="Stats.AzureCdnLogs.Common.Collect.ContentType" for the destination file./></param>
/// <param name="token">A <see cref="System.Threading.CancellationToken"/> to be used for cancelling the operation.</param>
/// <returns>A collection of exceptions if any.</returns>
public virtual async Task <AggregateException> TryProcessAsync(int maxFileCount, Func <string, string> fileNameTransform, ContentType sourceContentType, ContentType destinationContentType, CancellationToken token)
{
ConcurrentBag <Exception> exceptions = new ConcurrentBag <Exception>();
try
{
var files = await _source.GetFilesAsync(maxFileCount, token);
var parallelResult = Parallel.ForEach(files, (file) =>
{
if (token.IsCancellationRequested)
{
return;
}
var lockResult = _source.TakeLockAsync(file, token).Result;
if (lockResult.Item1 /*lockResult*/)
{
using (var inputStream = _source.OpenReadAsync(file, sourceContentType, token).Result)
{
var writeAction = VerifyStreamInternalAsync(file, sourceContentType, token).
ContinueWith(t =>
{
//if validation failed clean the file to not continue processing over and over
if (!t.Result)
{
throw new ApplicationException($"File {file} failed validation.");
}
_destination.WriteAsync(inputStream, ProcessLogStream, fileNameTransform(file.Segments.Last()), destinationContentType, token).Wait();
}).
ContinueWith(t =>
{
AddException(exceptions, t.Exception);
return(_source.CleanAsync(file, onError: t.IsFaulted, token: token).Result);
}).
ContinueWith(t =>
{
AddException(exceptions, t.Exception);
return(_source.ReleaseLockAsync(file, token).Result);
}).
ContinueWith(t =>
{
AddException(exceptions, t.Exception);
return(t.Result);
}).Result;
}
}
//log any exceptions from the renewlease task if faulted
//if the task is still running at this moment any future failure would not matter
if (lockResult.Item2 != null && lockResult.Item2.IsFaulted)
{
AddException(exceptions, lockResult.Item2.Exception);
}
});
}
catch (Exception e)
{
AddException(exceptions, e);
}
return(exceptions.Count() > 0 ? new AggregateException(exceptions.ToArray()) : null);
}
private async Task TryProcessBlobAsync(Uri file, Func <string, string> fileNameTransform, ContentType sourceContentType, ContentType destinationContentType, ConcurrentBag <Exception> exceptions, CancellationToken token)
{
_logger.LogInformation("TryProcessAsync: {File} ", file.AbsoluteUri);
if (token.IsCancellationRequested)
{
_logger.LogInformation("TryProcessAsync: The operation was cancelled.");
}
try
{
using (var lockResult = _source.TakeLockAsync(file, token).Result)
{
var blobOperationToken = lockResult.BlobOperationToken.Token;
if (lockResult.LockIsTaken /*lockResult*/)
{
using (var inputStream = await _source.OpenReadAsync(file, sourceContentType, blobOperationToken))
{
var blobToDeadLetter = !await VerifyStreamInternalAsync(file, sourceContentType, blobOperationToken);
// If verification passed continue with the rest of the action
// If not just move the blob to deadletter
if (!blobToDeadLetter)
{
var writeOperationResult = await _destination.TryWriteAsync(inputStream, ProcessLogStream, fileNameTransform(file.Segments.Last()), destinationContentType, blobOperationToken);
blobToDeadLetter = writeOperationResult.OperationException != null;
}
await _source.TryCleanAsync(lockResult, onError : blobToDeadLetter, token : blobOperationToken);
await _source.TryReleaseLockAsync(lockResult, token : blobOperationToken);
}
}
}
}
catch (Exception exception)
{
// Add any exceptions
AddException(exceptions, exception);
}
}
