/// <summary>
/// Try to process the files from the source.
/// After processing the file is cleaned. This means it wil be moved either to a archive or a deadletter container.
/// </summary>
/// <param name="maxFileCount">Only max this number of files will be processed at once.</param>
/// <param name="fileNameTransform">A Func to be used to generate the output file name fro the input filename.</param>
/// <param name="sourceContentType">The <see cref="Stats.AzureCdnLogs.Common.Collect.ContentType" for the source file./></param>
/// <param name="destinationContentType">The <see cref="Stats.AzureCdnLogs.Common.Collect.ContentType" for the destination file./></param>
/// <param name="token">A <see cref="System.Threading.CancellationToken"/> to be used for cancelling the operation.</param>
/// <returns>A collection of exceptions if any.</returns>
public virtual async Task <AggregateException> TryProcessAsync(int maxFileCount, Func <string, string> fileNameTransform, ContentType sourceContentType, ContentType destinationContentType, CancellationToken token)
{
ConcurrentBag <Exception> exceptions = new ConcurrentBag <Exception>();
try
{
var files = await _source.GetFilesAsync(maxFileCount, token);
var parallelResult = Parallel.ForEach(files, (file) =>
{
if (token.IsCancellationRequested)
{
return;
}
var lockResult = _source.TakeLockAsync(file, token).Result;
if (lockResult.Item1 /*lockResult*/)
{
using (var inputStream = _source.OpenReadAsync(file, sourceContentType, token).Result)
{
var writeAction = VerifyStreamInternalAsync(file, sourceContentType, token).
ContinueWith(t =>
{
//if validation failed clean the file to not continue processing over and over
if (!t.Result)
{
throw new ApplicationException($"File {file} failed validation.");
}
_destination.WriteAsync(inputStream, ProcessLogStream, fileNameTransform(file.Segments.Last()), destinationContentType, token).Wait();
}).
ContinueWith(t =>
{
AddException(exceptions, t.Exception);
return(_source.CleanAsync(file, onError: t.IsFaulted, token: token).Result);
}).
ContinueWith(t =>
{
AddException(exceptions, t.Exception);
return(_source.ReleaseLockAsync(file, token).Result);
}).
ContinueWith(t =>
{
AddException(exceptions, t.Exception);
return(t.Result);
}).Result;
}
}
//log any exceptions from the renewlease task if faulted
//if the task is still running at this moment any future failure would not matter
if (lockResult.Item2 != null && lockResult.Item2.IsFaulted)
{
AddException(exceptions, lockResult.Item2.Exception);
}
});
}
catch (Exception e)
{
AddException(exceptions, e);
}
return(exceptions.Count() > 0 ? new AggregateException(exceptions.ToArray()) : null);
}
