// Wraps our use of the proxy client library
private async Task GetFileAsync(ContainerItem ticketedItem, string tmpDownloadPath, CancellationToken cancellationToken)
{
// Will get doubled on each attempt.
TimeSpan timeBetweenRetries = ContainerFetchEngineOptions.RetryInterval;
for (int triesRemaining = ContainerFetchEngineOptions.RetryLimit; ; triesRemaining--)
{
bool lastAttempt = (triesRemaining == 0);
timeBetweenRetries += timeBetweenRetries;
// Delete the tmp file inbetween attempts
FileSystemManager.DeleteFile(tmpDownloadPath);
try
{
Task <Stream> getFileTask = Provider.GetFileTask(ticketedItem, cancellationToken);
ExecutionLogger.Debug(StringUtil.Format("Fetching contents of file {0}", tmpDownloadPath));
await getFileTask.ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
return;
}
ExecutionLogger.Debug(StringUtil.Format("Writing contents of file {0} to disk", tmpDownloadPath));
using (Stream stream = await getFileTask.ConfigureAwait(false))
{
await FileSystemManager.WriteStreamToFile(stream, tmpDownloadPath, ContainerFetchEngineOptions.DownloadBufferSize, cancellationToken);
}
ExecutionLogger.Debug(StringUtil.Format("Finished writing contents of file {0} to disk", tmpDownloadPath));
break;
}
catch (Exception exception)
{
if (exception is AggregateException)
{
exception = ((AggregateException)exception).Flatten().InnerException;
}
if (lastAttempt)
{
throw new Exception(StringUtil.Loc("RMErrorDownloadingContainerItem", tmpDownloadPath, exception));
}
lock (_lock)
{
ExecutionLogger.Warning(StringUtil.Loc("RMReAttemptingDownloadOfContainerItem", tmpDownloadPath, exception));
}
}
// "Sleep" in between attempts. (Can't await inside a catch clause.)
await Task.Delay(timeBetweenRetries);
}
}
// Wraps our use of the proxy client library
private async Task GetFileAsync(ContainerItem ticketedItem, string tmpDownloadPath, CancellationToken cancellationToken)
{
// Will get doubled on each attempt.
TimeSpan timeBetweenRetries = ContainerFetchEngineOptions.RetryInterval;
for (int triesRemaining = ContainerFetchEngineOptions.RetryLimit; ; triesRemaining--)
{
bool lastAttempt = (triesRemaining == 0);
timeBetweenRetries += timeBetweenRetries;
// Delete the tmp file inbetween attempts
FileSystemManager.DeleteFile(tmpDownloadPath);
try
{
Task <Stream> getFileTask = Provider.GetFileTask(ticketedItem, cancellationToken);
Task timeoutTask = Task.Delay(ContainerFetchEngineOptions.GetFileAsyncTimeout, cancellationToken);
// Wait for GetFileAsync or the timeout to elapse.
await Task.WhenAny(getFileTask, timeoutTask).ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
return;
}
if (!getFileTask.IsCompleted)
{
throw new TimeoutException(
StringUtil.Loc("RMGetFileAsyncTimedOut", GetFileAsyncTimeoutMinutes));
}
using (Stream stream = await getFileTask.ConfigureAwait(false))
{
await FileSystemManager.WriteStreamToFile(stream, tmpDownloadPath, cancellationToken);
}
break;
}
catch (Exception exception)
{
if (lastAttempt)
{
throw new Exception(StringUtil.Loc("RMErrorDownloadingContainerItem", tmpDownloadPath, exception));
}
lock (_lock)
{
ExecutionLogger.Warning(StringUtil.Loc("RMReAttemptingDownloadOfContainerItem", tmpDownloadPath, exception.Message));
}
}
// "Sleep" inbetween attempts. (Can't await inside a catch clause.)
await Task.Delay(timeBetweenRetries);
}
}
protected async Task FetchItemsAsync(IEnumerable <ContainerItem> containerItems, CancellationToken token)
{
ArgUtil.NotNull(containerItems, nameof(containerItems));
var itemsToDownload = new List <ContainerItem>();
foreach (ContainerItem item in containerItems)
{
if (item.ItemType == ItemType.Folder)
{
string localDirectory = ConvertToLocalPath(item);
FileSystemManager.EnsureDirectoryExists(localDirectory);
}
else if (item.ItemType == ItemType.File)
{
string localPath = ConvertToLocalPath(item);
ExecutionLogger.Debug(StringUtil.Format("[File] {0} => {1}", item.Path, localPath));
_totalFiles++;
if (item.FileLength == 0)
{
CreateEmptyFile(localPath);
_newEmptyFiles++;
}
else
{
itemsToDownload.Add(item);
}
}
else
{
throw new NotSupportedException(StringUtil.Loc("RMContainerItemNotSupported", item.ItemType));
}
}
if (_totalFiles == 0)
{
ExecutionLogger.Warning(StringUtil.Loc("RMArtifactEmpty"));
}
if (itemsToDownload.Count > 0)
{
using (var cancellationTokenSource = new CancellationTokenSource())
using (var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, cancellationTokenSource.Token))
using (var downloadThrottle = new SemaphoreSlim(ContainerFetchEngineOptions.ParallelDownloadLimit))
{
CancellationToken cancellationToken = linkedTokenSource.Token;
// Used to limit the number of concurrent downloads.
Stopwatch watch = Stopwatch.StartNew();
LinkedList <Task> remainingDownloads = new LinkedList <Task>();
foreach (ContainerItem ticketedItem in itemsToDownload)
{
_bytesDownloaded += ticketedItem.FileLength;
Task downloadTask = DownloadItemAsync(downloadThrottle, ticketedItem, cancellationToken);
if (downloadTask.IsCompleted)
{
// don't wait to throw for faulted tasks.
await downloadTask.ConfigureAwait(false);
}
else
{
remainingDownloads.AddLast(downloadTask);
}
}
try
{
// Monitor and log the progress of the download tasks if they take over a few seconds.
await LogProgressAsync(remainingDownloads).ConfigureAwait(false);
}
catch (Exception)
{
cancellationTokenSource.Cancel();
await Task.WhenAll(remainingDownloads);
throw;
}
_elapsedDownloadTime += watch.Elapsed;
}
_downloadedFiles += itemsToDownload.Count;
}
}
private async Task LogProgressAsync(LinkedList <Task> remainingTasks)
{
Stopwatch watch = Stopwatch.StartNew();
// Log progress until all downloads complete
while (remainingTasks.Any())
{
Task delayTask = Task.Delay(ProgressInterval);
if (remainingTasks.Count < 20)
{
// temporarily add the delay task.
remainingTasks.AddLast(delayTask);
// wait for the delay task or a download to complete.
// Task.WhenAny is probably an O(n) operation, so we only do this if there's not many downloads remaining.
await Task.WhenAny(remainingTasks).ConfigureAwait(false);
// remove the delay task.
remainingTasks.RemoveLast();
}
else
{
// go do something else for 5 seconds.
await delayTask.ConfigureAwait(false);
}
// remove any download tasks that completed.
LinkedListNode <Task> task = remainingTasks.First;
while (task != null)
{
LinkedListNode <Task> nextTask = task.Next;
if (task.Value.IsCompleted)
{
// don't wait to throw for faulted tasks.
await task.Value.ConfigureAwait(false);
remainingTasks.Remove(task);
}
task = nextTask;
}
// check how many downloads remain.
if (remainingTasks.Count > 0)
{
if (watch.Elapsed > ProgressInterval)
{
ExecutionLogger.Output(StringUtil.Loc("RMRemainingDownloads", remainingTasks.Count));
watch.Restart();
}
if (remainingTasks.Count != _previousRemainingTaskCount)
{
_lastTaskCompletionTime = DateTime.UtcNow;
_previousRemainingTaskCount = remainingTasks.Count;
}
TimeSpan timeSinceLastTaskCompletion = DateTime.UtcNow - _lastTaskCompletionTime;
TimeSpan timeSinceLastDiag = DateTime.UtcNow - _lastTaskDiagTime;
if (timeSinceLastTaskCompletion > TaskDiagThreshold &&
timeSinceLastDiag > TaskDiagThreshold)
{
var taskStates = remainingTasks.GroupBy(dt => dt.Status);
lock (_lock)
{
ExecutionLogger.Warning(StringUtil.Loc("RMDownloadTaskCompletedStatus", (int)timeSinceLastTaskCompletion.TotalMinutes));
foreach (IGrouping <TaskStatus, Task> group in taskStates)
{
ExecutionLogger.Warning(StringUtil.Loc("RMDownloadTaskStates", group.Key, group.Count()));
}
}
_lastTaskDiagTime = DateTime.UtcNow;
}
}
}
}
protected async Task FetchItemsAsync(IEnumerable <ContainerItem> containerItems)
{
var itemsToDownload = new List <ContainerItem>();
foreach (ContainerItem item in containerItems)
{
if (item.ItemType == ItemType.Folder)
{
string localDirectory = ConvertToLocalPath(item);
FileSystemManager.EnsureDirectoryExists(localDirectory);
}
else if (item.ItemType == ItemType.File)
{
string localPath = ConvertToLocalPath(item);
ExecutionLogger.Info(StringUtil.Loc("RMCopyingFile", item.Path, localPath));
_totalFiles++;
if (item.FileLength == 0)
{
CreateEmptyFile(localPath);
_newEmptyFiles++;
}
else
{
itemsToDownload.Add(item);
}
}
else
{
throw new NotSupportedException(StringUtil.Loc("RMContainerItemNotSupported", item.ItemType));
}
}
if (_totalFiles == 0)
{
ExecutionLogger.Warning(StringUtil.Loc("RMArtifactEmpty"));
}
if (itemsToDownload.Count > 0)
{
// Used to limit the number of concurrent downloads.
SemaphoreSlim downloadThrottle = new SemaphoreSlim(ContainerFetchEngineOptions.ParallelDownloadLimit);
Stopwatch watch = Stopwatch.StartNew();
LinkedList <Task> remainingDownloads = new LinkedList <Task>();
foreach (ContainerItem ticketedItem in itemsToDownload)
{
_bytesDownloaded += ticketedItem.FileLength;
Task downloadTask = DownloadItemAsync(downloadThrottle, ticketedItem);
if (downloadTask.IsCompleted)
{
// don't wait to throw for faulted tasks.
await downloadTask.ConfigureAwait(false);
}
else
{
remainingDownloads.AddLast(downloadTask);
}
}
// Monitor and log the progress of the download tasks if they take over a few seconds.
await LogProgressAsync(remainingDownloads).ConfigureAwait(false);
_elapsedDownloadTime += watch.Elapsed;
}
_downloadedFiles += itemsToDownload.Count;
}