// Wraps our use of the proxy client library
private async Task GetFileAsync(ContainerItem ticketedItem, string tmpDownloadPath, CancellationToken cancellationToken)
{
// Will get doubled on each attempt.
TimeSpan timeBetweenRetries = ContainerFetchEngineOptions.RetryInterval;
for (int triesRemaining = ContainerFetchEngineOptions.RetryLimit; ; triesRemaining--)
{
bool lastAttempt = (triesRemaining == 0);
timeBetweenRetries += timeBetweenRetries;
// Delete the tmp file inbetween attempts
FileSystemManager.DeleteFile(tmpDownloadPath);
try
{
Task <Stream> getFileTask = Provider.GetFileTask(ticketedItem, cancellationToken);
ExecutionLogger.Debug(StringUtil.Format("Fetching contents of file {0}", tmpDownloadPath));
await getFileTask.ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
return;
}
ExecutionLogger.Debug(StringUtil.Format("Writing contents of file {0} to disk", tmpDownloadPath));
using (Stream stream = await getFileTask.ConfigureAwait(false))
{
await FileSystemManager.WriteStreamToFile(stream, tmpDownloadPath, ContainerFetchEngineOptions.DownloadBufferSize, cancellationToken);
}
ExecutionLogger.Debug(StringUtil.Format("Finished writing contents of file {0} to disk", tmpDownloadPath));
break;
}
catch (Exception exception)
{
if (exception is AggregateException)
{
exception = ((AggregateException)exception).Flatten().InnerException;
}
if (lastAttempt)
{
throw new Exception(StringUtil.Loc("RMErrorDownloadingContainerItem", tmpDownloadPath, exception));
}
lock (_lock)
{
ExecutionLogger.Warning(StringUtil.Loc("RMReAttemptingDownloadOfContainerItem", tmpDownloadPath, exception));
}
}
// "Sleep" in between attempts. (Can't await inside a catch clause.)
await Task.Delay(timeBetweenRetries);
}
}
private async Task DownloadItemImplAsync(
SemaphoreSlim downloadThrottle,
ContainerItem ticketedItem,
string downloadPath,
CancellationToken cancellationToken)
{
try
{
lock (_lock)
{
ExecutionLogger.Debug(StringUtil.Format("Acquiring semaphore to download file {0}", downloadPath));
}
await downloadThrottle.WaitAsync().ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
return;
}
// Download the file content to a temp file on the same drive.
// Assumption: the build won't produce files ending in .download.
string tmpDownloadPath = downloadPath + ".download";
FileSystemManager.EnsureParentDirectory(tmpDownloadPath);
FileSystemManager.DeleteFile(downloadPath);
lock (_lock)
{
ExecutionLogger.Output(StringUtil.Loc("RMDownloadStartDownloadOfFile", downloadPath));
}
await GetFileAsync(ticketedItem, tmpDownloadPath, cancellationToken).ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
return;
}
// With the content successfully downloaded, move the tmp file to its permanent location.
FileSystemManager.MoveFile(tmpDownloadPath, downloadPath);
}
finally
{
downloadThrottle.Release();
}
}
protected async Task FetchItemsAsync(IEnumerable <ContainerItem> containerItems, CancellationToken token)
{
ArgUtil.NotNull(containerItems, nameof(containerItems));
var itemsToDownload = new List <ContainerItem>();
foreach (ContainerItem item in containerItems)
{
if (item.ItemType == ItemType.Folder)
{
string localDirectory = ConvertToLocalPath(item);
FileSystemManager.EnsureDirectoryExists(localDirectory);
}
else if (item.ItemType == ItemType.File)
{
string localPath = ConvertToLocalPath(item);
ExecutionLogger.Debug(StringUtil.Format("[File] {0} => {1}", item.Path, localPath));
_totalFiles++;
if (item.FileLength == 0)
{
CreateEmptyFile(localPath);
_newEmptyFiles++;
}
else
{
itemsToDownload.Add(item);
}
}
else
{
throw new NotSupportedException(StringUtil.Loc("RMContainerItemNotSupported", item.ItemType));
}
}
if (_totalFiles == 0)
{
ExecutionLogger.Warning(StringUtil.Loc("RMArtifactEmpty"));
}
if (itemsToDownload.Count > 0)
{
using (var cancellationTokenSource = new CancellationTokenSource())
using (var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, cancellationTokenSource.Token))
using (var downloadThrottle = new SemaphoreSlim(ContainerFetchEngineOptions.ParallelDownloadLimit))
{
CancellationToken cancellationToken = linkedTokenSource.Token;
// Used to limit the number of concurrent downloads.
Stopwatch watch = Stopwatch.StartNew();
LinkedList <Task> remainingDownloads = new LinkedList <Task>();
foreach (ContainerItem ticketedItem in itemsToDownload)
{
_bytesDownloaded += ticketedItem.FileLength;
Task downloadTask = DownloadItemAsync(downloadThrottle, ticketedItem, cancellationToken);
if (downloadTask.IsCompleted)
{
// don't wait to throw for faulted tasks.
await downloadTask.ConfigureAwait(false);
}
else
{
remainingDownloads.AddLast(downloadTask);
}
}
try
{
// Monitor and log the progress of the download tasks if they take over a few seconds.
await LogProgressAsync(remainingDownloads).ConfigureAwait(false);
}
catch (Exception)
{
cancellationTokenSource.Cancel();
await Task.WhenAll(remainingDownloads);
throw;
}
_elapsedDownloadTime += watch.Elapsed;
}
_downloadedFiles += itemsToDownload.Count;
}
}