private async Task <DownloadResult> ParallelDownloadAsync(RunnerActionPluginExecutionContext context, IReadOnlyList <DownloadInfo> files, int concurrentDownloads, CancellationToken token)
{
// return files that fail to download
var downloadResult = new DownloadResult();
// nothing needs to download
if (files.Count == 0)
{
return(downloadResult);
}
// ensure the file download queue is empty.
if (!_fileDownloadQueue.IsEmpty)
{
throw new ArgumentOutOfRangeException(nameof(_fileDownloadQueue));
}
// enqueue file into download queue.
foreach (var file in files)
{
_fileDownloadQueue.Enqueue(file);
}
// Start download monitor task.
_downloadFilesProcessed = 0;
_downloadFinished = new TaskCompletionSource <int>();
Task downloadMonitor = DownloadReportingAsync(context, files.Count(), token);
// Start parallel download tasks.
List <Task <DownloadResult> > parallelDownloadingTasks = new List <Task <DownloadResult> >();
for (int downloader = 0; downloader < concurrentDownloads; downloader++)
{
parallelDownloadingTasks.Add(DownloadAsync(context, downloader, token));
}
// Wait for parallel download finish.
await Task.WhenAll(parallelDownloadingTasks);
foreach (var downloadTask in parallelDownloadingTasks)
{
// record all failed files.
downloadResult.AddDownloadResult(await downloadTask);
}
// Stop monitor task;
_downloadFinished.TrySetResult(0);
await downloadMonitor;
return(downloadResult);
}
public void AddDownloadResult(DownloadResult resultToAdd)
{
this.FailedFiles.AddRange(resultToAdd.FailedFiles);
}
public async Task DownloadFromContainerAsync(
RunnerActionPluginExecutionContext context,
String destination,
CancellationToken cancellationToken)
{
// Find out all container items need to be processed
List <FileContainerItem> containerItems = new List <FileContainerItem>();
int retryCount = 0;
while (retryCount < 3)
{
try
{
containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId,
_projectId,
_containerPath,
cancellationToken : cancellationToken);
break;
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
context.Debug($"Container query has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}");
context.Debug(ex.ToString());
}
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
context.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
if (containerItems.Count == 0)
{
context.Output($"There is nothing under #/{_containerId}/{_containerPath}");
return;
}
// container items will include both folders, files and even file with zero size
// Create all required empty folders and emptry files, gather a list of files that we need to download from server.
int foldersCreated = 0;
int emptryFilesCreated = 0;
List <DownloadInfo> downloadFiles = new List <DownloadInfo>();
foreach (var item in containerItems.OrderBy(x => x.Path))
{
if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}");
}
var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/');
var localPath = Path.Combine(destination, localRelativePath);
if (item.ItemType == ContainerItemType.Folder)
{
context.Debug($"Ensure folder exists: {localPath}");
Directory.CreateDirectory(localPath);
foldersCreated++;
}
else if (item.ItemType == ContainerItemType.File)
{
if (item.FileLength == 0)
{
context.Debug($"Create empty file at: {localPath}");
var parentDirectory = Path.GetDirectoryName(localPath);
Directory.CreateDirectory(parentDirectory);
IOUtil.DeleteFile(localPath);
using (new FileStream(localPath, FileMode.Create))
{
}
emptryFilesCreated++;
}
else
{
context.Debug($"Prepare download {item.Path} to {localPath}");
downloadFiles.Add(new DownloadInfo(item.Path, localPath));
}
}
else
{
throw new NotSupportedException(item.ItemType.ToString());
}
}
if (foldersCreated > 0)
{
context.Output($"{foldersCreated} folders created.");
}
if (emptryFilesCreated > 0)
{
context.Output($"{emptryFilesCreated} empty files created.");
}
if (downloadFiles.Count == 0)
{
context.Output($"There is nothing to download");
return;
}
// Start multi-task to download all files.
using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// try download all files for the first time.
DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
if (downloadResult.FailedFiles.Count == 0)
{
// all files have been download succeed.
context.Output($"{downloadFiles.Count} files download succeed.");
return;
}
else
{
context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute.");
}
// Delay 1 min then retry failed files.
for (int timer = 60; timer > 0; timer -= 5)
{
context.Output($"Retry file download after {timer} seconds.");
await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
}
// Retry download all failed files.
context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload.");
DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
if (retryDownloadResult.FailedFiles.Count == 0)
{
// all files have been download succeed after retry.
context.Output($"{downloadResult.FailedFiles} files download succeed after retry.");
return;
}
else
{
throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry.");
}
}
}
