public void AddUploadResult(UploadResult resultToAdd) { this.FailedFiles.AddRange(resultToAdd.FailedFiles); this.TotalFileSizeUploaded += resultToAdd.TotalFileSizeUploaded; }
public async Task <long> CopyToContainerAsync( IAsyncCommandContext context, String source, CancellationToken cancellationToken) { //set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2); //context.Output($"Max Concurrent Uploads {maxConcurrentUploads}"); List <String> files; if (File.Exists(source)) { files = new List <String>() { source }; _sourceParentDirectory = Path.GetDirectoryName(source); } else { files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList(); _sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); } context.Output(StringUtil.Loc("TotalUploadFiles", files.Count())); using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { // hook up reporting event from file container client. _fileContainerHttpClient.UploadFileReportTrace += UploadFileTraceReportReceived; _fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived; try { // try upload all files for the first time. UploadResult uploadResult = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token); if (uploadResult.FailedFiles.Count == 0) { // all files have been upload succeed. context.Output(StringUtil.Loc("FileUploadSucceed")); return(uploadResult.TotalFileSizeUploaded); } else { context.Output(StringUtil.Loc("FileUploadFailedRetryLater", uploadResult.FailedFiles.Count)); } // Delay 1 min then retry failed files. for (int timer = 60; timer > 0; timer -= 5) { context.Output(StringUtil.Loc("FileUploadRetryInSecond", timer)); await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token); } // Retry upload all failed files. context.Output(StringUtil.Loc("FileUploadRetry", uploadResult.FailedFiles.Count)); UploadResult retryUploadResult = await ParallelUploadAsync(context, uploadResult.FailedFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token); if (retryUploadResult.FailedFiles.Count == 0) { // all files have been upload succeed after retry. context.Output(StringUtil.Loc("FileUploadRetrySucceed")); return(uploadResult.TotalFileSizeUploaded + retryUploadResult.TotalFileSizeUploaded); } else { throw new Exception(StringUtil.Loc("FileUploadFailedAfterRetry")); } } finally { _fileContainerHttpClient.UploadFileReportTrace -= UploadFileTraceReportReceived; _fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived; } } }
private async Task <UploadResult> ParallelUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token) { // return files that fail to upload and total artifact size var uploadResult = new UploadResult(); // nothing needs to upload if (files.Count == 0) { return(uploadResult); } var uploadToBlob = String.Equals(context.GetVariableValueOrDefault("agent.UploadBuildArtifactsToBlob"), "true", StringComparison.InvariantCultureIgnoreCase); if (uploadToBlob) { var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase); var(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance .CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, token); _dedupClient = dedupClient; _blobTelemetry = clientTelemetry; } // ensure the file upload queue is empty. if (!_fileUploadQueue.IsEmpty) { throw new ArgumentOutOfRangeException(nameof(_fileUploadQueue)); } // enqueue file into upload queue. foreach (var file in files) { _fileUploadQueue.Enqueue(file); } // Start upload monitor task. _filesProcessed = 0; _uploadFinished = new TaskCompletionSource <int>(); _fileUploadTraceLog.Clear(); _fileUploadProgressLog.Clear(); Task uploadMonitor = ReportingAsync(context, files.Count(), _uploadCancellationTokenSource.Token); // Start parallel upload tasks. List <Task <UploadResult> > parallelUploadingTasks = new List <Task <UploadResult> >(); for (int uploader = 0; uploader < concurrentUploads; uploader++) { parallelUploadingTasks.Add(UploadAsync(context, uploader, uploadToBlob, _uploadCancellationTokenSource.Token)); } // Wait for parallel upload finish. await Task.WhenAll(parallelUploadingTasks); foreach (var uploadTask in parallelUploadingTasks) { // record all failed files. uploadResult.AddUploadResult(await uploadTask); } // Stop monitor task; _uploadFinished.TrySetResult(0); await uploadMonitor; return(uploadResult); }