public async Task <Stream> GetFileTask(ContainerItem ticketedItem, CancellationToken cancellationToken)
{
ArgUtil.NotNull(ticketedItem, nameof(ticketedItem));
this._executionContext.Debug(StringUtil.Format("Get file container client for file {0}", ticketedItem.Path));
VssConnection vssConnection = await GetVssConnection();
FileContainerHttpClient fileContainer = null;
try
{
fileContainer = vssConnection.GetClient <FileContainerHttpClient>();
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, vssConnection.Uri.ToString(), this._executionContext.Error);
throw;
}
this._executionContext.Debug(StringUtil.Format("Start fetch file stream from filecontainer service for file {0}", ticketedItem.Path));
Stream stream = await fileContainer.DownloadFileAsync(
ticketedItem.ContainerId,
ticketedItem.Path,
cancellationToken,
scopeIdentifier : ticketedItem.ScopeIdentifier);
this._executionContext.Debug(StringUtil.Format("Finished fetch file stream from filecontainer service for file {0}", ticketedItem.Path));
return(stream);
}
private async Task <DownloadResult> DownloadAsync(RunnerActionPluginExecutionContext context, int downloaderId, CancellationToken token)
{
List <DownloadInfo> failedFiles = new List <DownloadInfo>();
Stopwatch downloadTimer = new Stopwatch();
while (_fileDownloadQueue.TryDequeue(out DownloadInfo fileToDownload))
{
token.ThrowIfCancellationRequested();
try
{
int retryCount = 0;
bool downloadFailed = false;
while (true)
{
try
{
context.Debug($"Start downloading file: '{fileToDownload.ItemPath}' (Downloader {downloaderId})");
downloadTimer.Restart();
using (FileStream fs = new FileStream(fileToDownload.LocalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var downloadStream = await _fileContainerHttpClient.DownloadFileAsync(_containerId, fileToDownload.ItemPath, token, _projectId))
{
await downloadStream.CopyToAsync(fs, _defaultCopyBufferSize, token);
await fs.FlushAsync(token);
downloadTimer.Stop();
context.Debug($"File: '{fileToDownload.LocalPath}' took {downloadTimer.ElapsedMilliseconds} milliseconds to finish download (Downloader {downloaderId})");
break;
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Debug($"Download has been cancelled while downloading {fileToDownload.ItemPath}. (Downloader {downloaderId})");
throw;
}
catch (Exception ex)
{
retryCount++;
context.Warning($"Fail to download '{fileToDownload.ItemPath}', error: {ex.Message} (Downloader {downloaderId})");
context.Debug(ex.ToString());
}
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
context.Warning($"Back off {backOff.TotalSeconds} seconds before retry. (Downloader {downloaderId})");
await Task.Delay(backOff);
}
else
{
// upload still failed after 3 tries.
downloadFailed = true;
break;
}
}
if (downloadFailed)
{
// tracking file that failed to download.
failedFiles.Add(fileToDownload);
}
Interlocked.Increment(ref _downloadFilesProcessed);
}
catch (Exception ex)
{
// We should never
context.Error($"Error '{ex.Message}' when downloading file '{fileToDownload}'. (Downloader {downloaderId})");
throw;
}
}
return(new DownloadResult(failedFiles));
}