private async Task <UploadResult> UploadAsync(IAsyncCommandContext context, int uploaderId, bool uploadToBlob, CancellationToken token)
{
List <string> failedFiles = new List <string>();
long uploadedSize = 0;
string fileToUpload;
Stopwatch uploadTimer = new Stopwatch();
while (_fileUploadQueue.TryDequeue(out fileToUpload))
{
token.ThrowIfCancellationRequested();
try
{
string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
uploadTimer.Restart();
bool catchExceptionDuringUpload = false;
HttpResponseMessage response = null;
long uploadLength = 0;
try
{
if (uploadToBlob)
{
var result = await UploadToBlobStore(context, fileToUpload, token);
var retryHelper = new RetryHelper(context);
response = await retryHelper.Retry(async() => await _fileContainerHttpClient.CreateItemForArtifactUpload(_containerId, itemPath, _projectId,
result.dedupId.ValueString, (long)result.length, token),
(retryCounter) => (int)Math.Pow(retryCounter, 2) * 5,
(exception) => true);
uploadLength = (long)result.length;
}
else
{
using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read, FileShare.Read))
{
response = await _fileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, cancellationToken : token, chunkSize : 4 * 1024 * 1024);
uploadLength = fs.Length;
}
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Output(StringUtil.Loc("FileUploadCancelled", fileToUpload));
if (response != null)
{
response.Dispose();
response = null;
}
throw;
}
catch (Exception ex)
{
catchExceptionDuringUpload = true;
context.Output(StringUtil.Loc("FileUploadFailed", fileToUpload, ex.Message));
context.Output(ex.ToString());
}
uploadTimer.Stop();
if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created))
{
if (response != null)
{
context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, fileToUpload, itemPath));
}
// output detail upload trace for the file.
ConcurrentQueue <string> logQueue;
if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue))
{
context.Output(StringUtil.Loc("FileUploadDetailTrace", itemPath));
string message;
while (logQueue.TryDequeue(out message))
{
context.Output(message);
}
}
// tracking file that failed to upload.
failedFiles.Add(fileToUpload);
}
else
{
context.Debug(StringUtil.Loc("FileUploadFinish", fileToUpload, uploadTimer.ElapsedMilliseconds));
uploadedSize += uploadLength;
// debug detail upload trace for the file.
ConcurrentQueue <string> logQueue;
if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue))
{
context.Debug($"Detail upload trace for file: {itemPath}");
string message;
while (logQueue.TryDequeue(out message))
{
context.Debug(message);
}
}
}
if (response != null)
{
response.Dispose();
response = null;
}
Interlocked.Increment(ref _filesProcessed);
}
catch (Exception ex)
{
context.Output(StringUtil.Loc("FileUploadFileOpenFailed", ex.Message, fileToUpload));
throw;
}
}
return(new UploadResult(failedFiles, uploadedSize));
}
private async Task <UploadResult> AssociateAsync(IAsyncCommandContext context, ConcurrentQueue <BlobFileInfo> associateQueue, CancellationToken token)
{
var uploadResult = new UploadResult();
var retryHelper = new RetryHelper(context);
var uploadTimer = new Stopwatch();
while (associateQueue.TryDequeue(out var file))
{
uploadTimer.Restart();
string itemPath = (_containerPath.TrimEnd('/') + "/" + file.Path.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
bool catchExceptionDuringUpload = false;
HttpResponseMessage response = null;
try
{
if (file.Success)
{
var length = (long)file.Node.TransitiveContentBytes;
response = await retryHelper.Retry(async() => await _fileContainerHttpClient.CreateItemForArtifactUpload(_containerId, itemPath, _projectId,
file.DedupId.ValueString, length, token),
(retryCounter) => (int)Math.Pow(retryCounter, 2) * 5,
(exception) => true);
uploadResult.TotalFileSizeUploaded += length;
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Output(StringUtil.Loc("FileUploadCancelled", itemPath));
if (response != null)
{
response.Dispose();
}
throw;
}
catch (Exception ex)
{
catchExceptionDuringUpload = true;
context.Output(StringUtil.Loc("FileUploadFailed", itemPath, ex.Message));
context.Output(ex.ToString());
}
if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created) || !file.Success)
{
if (response != null)
{
context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, file.Path, itemPath));
}
if (!file.Success)
{
context.Output(StringUtil.Loc("FileContainerUploadFailedBlob", file.Path, itemPath));
}
// tracking file that failed to upload.
uploadResult.FailedFiles.Add(file.Path);
}
else
{
context.Debug(StringUtil.Loc("FileUploadFinish", file.Path, uploadTimer.ElapsedMilliseconds));
}
if (response != null)
{
response.Dispose();
}
Interlocked.Increment(ref _filesProcessed);
}
return(uploadResult);
}