private async Task <UploadResult> UploadAsync(IAsyncCommandContext context, int uploaderId, bool uploadToBlob, CancellationToken token) { List <string> failedFiles = new List <string>(); long uploadedSize = 0; string fileToUpload; Stopwatch uploadTimer = new Stopwatch(); while (_fileUploadQueue.TryDequeue(out fileToUpload)) { token.ThrowIfCancellationRequested(); try { string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/'); uploadTimer.Restart(); bool catchExceptionDuringUpload = false; HttpResponseMessage response = null; long uploadLength = 0; try { if (uploadToBlob) { var result = await UploadToBlobStore(context, fileToUpload, token); var retryHelper = new RetryHelper(context); response = await retryHelper.Retry(async() => await _fileContainerHttpClient.CreateItemForArtifactUpload(_containerId, itemPath, _projectId, result.dedupId.ValueString, (long)result.length, token), (retryCounter) => (int)Math.Pow(retryCounter, 2) * 5, (exception) => true); uploadLength = (long)result.length; } else { using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read, FileShare.Read)) { response = await _fileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, cancellationToken : token, chunkSize : 4 * 1024 * 1024); uploadLength = fs.Length; } } } catch (OperationCanceledException) when(token.IsCancellationRequested) { context.Output(StringUtil.Loc("FileUploadCancelled", fileToUpload)); if (response != null) { response.Dispose(); response = null; } throw; } catch (Exception ex) { catchExceptionDuringUpload = true; context.Output(StringUtil.Loc("FileUploadFailed", fileToUpload, ex.Message)); context.Output(ex.ToString()); } uploadTimer.Stop(); if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created)) { if (response != null) { context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, fileToUpload, itemPath)); } // output detail upload trace for the file. ConcurrentQueue <string> logQueue; if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue)) { context.Output(StringUtil.Loc("FileUploadDetailTrace", itemPath)); string message; while (logQueue.TryDequeue(out message)) { context.Output(message); } } // tracking file that failed to upload. failedFiles.Add(fileToUpload); } else { context.Debug(StringUtil.Loc("FileUploadFinish", fileToUpload, uploadTimer.ElapsedMilliseconds)); uploadedSize += uploadLength; // debug detail upload trace for the file. ConcurrentQueue <string> logQueue; if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue)) { context.Debug($"Detail upload trace for file: {itemPath}"); string message; while (logQueue.TryDequeue(out message)) { context.Debug(message); } } } if (response != null) { response.Dispose(); response = null; } Interlocked.Increment(ref _filesProcessed); } catch (Exception ex) { context.Output(StringUtil.Loc("FileUploadFileOpenFailed", ex.Message, fileToUpload)); throw; } } return(new UploadResult(failedFiles, uploadedSize)); }
private async Task <UploadResult> AssociateAsync(IAsyncCommandContext context, ConcurrentQueue <BlobFileInfo> associateQueue, CancellationToken token) { var uploadResult = new UploadResult(); var retryHelper = new RetryHelper(context); var uploadTimer = new Stopwatch(); while (associateQueue.TryDequeue(out var file)) { uploadTimer.Restart(); string itemPath = (_containerPath.TrimEnd('/') + "/" + file.Path.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/'); bool catchExceptionDuringUpload = false; HttpResponseMessage response = null; try { if (file.Success) { var length = (long)file.Node.TransitiveContentBytes; response = await retryHelper.Retry(async() => await _fileContainerHttpClient.CreateItemForArtifactUpload(_containerId, itemPath, _projectId, file.DedupId.ValueString, length, token), (retryCounter) => (int)Math.Pow(retryCounter, 2) * 5, (exception) => true); uploadResult.TotalFileSizeUploaded += length; } } catch (OperationCanceledException) when(token.IsCancellationRequested) { context.Output(StringUtil.Loc("FileUploadCancelled", itemPath)); if (response != null) { response.Dispose(); } throw; } catch (Exception ex) { catchExceptionDuringUpload = true; context.Output(StringUtil.Loc("FileUploadFailed", itemPath, ex.Message)); context.Output(ex.ToString()); } if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created) || !file.Success) { if (response != null) { context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, file.Path, itemPath)); } if (!file.Success) { context.Output(StringUtil.Loc("FileContainerUploadFailedBlob", file.Path, itemPath)); } // tracking file that failed to upload. uploadResult.FailedFiles.Add(file.Path); } else { context.Debug(StringUtil.Loc("FileUploadFinish", file.Path, uploadTimer.ElapsedMilliseconds)); } if (response != null) { response.Dispose(); } Interlocked.Increment(ref _filesProcessed); } return(uploadResult); }