public static async Task UploadArtifactAsync( IAsyncCommandContext context, VssConnection connection, Guid projectId, long containerId, string containerPath, int buildId, string name, string jobId, Dictionary <string, string> propertiesDictionary, string source, CancellationToken cancellationToken) { var fileContainerHelper = new FileContainerServer(connection, projectId, containerId, containerPath); var size = await fileContainerHelper.CopyToContainerAsync(context, source, cancellationToken); propertiesDictionary.Add(ArtifactUploadEventProperties.ArtifactSize, size.ToString()); var fileContainerFullPath = StringUtil.Format($"#/{containerId}/{containerPath}"); context.Output(StringUtil.Loc("UploadToFileContainer", source, fileContainerFullPath)); var buildHelper = context.GetHostContext().GetService <IBuildServer>(); await buildHelper.ConnectAsync(connection); var artifact = await buildHelper.AssociateArtifactAsync(buildId, projectId, name, jobId, ArtifactResourceTypes.Container, fileContainerFullPath, propertiesDictionary, cancellationToken); context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId)); }
private async Task UploadArtifactAsync( IAsyncCommandContext context, Uri projectCollection, VssCredentials credentials, Guid projectId, long containerId, string containerPath, int buildId, string name, Dictionary <string, string> propertiesDictionary, string source, CancellationToken cancellationToken) { FileContainerServer fileContainerHelper = new FileContainerServer(projectCollection, credentials, projectId, containerId, containerPath); await fileContainerHelper.CopyToContainerAsync(context, source, cancellationToken); string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{containerPath}"); context.Output(StringUtil.Loc("UploadToFileContainer", source, fileContainerFullPath)); BuildServer buildHelper = new BuildServer(projectCollection, credentials, projectId); var artifact = await buildHelper.AssociateArtifact(buildId, name, WellKnownArtifactResourceTypes.Container, fileContainerFullPath, propertiesDictionary, cancellationToken); context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId)); }
private async Task ReportingAsync(IAsyncCommandContext context, int totalFiles, CancellationToken token) { int traceInterval = 0; while (!_uploadFinished.Task.IsCompleted && !token.IsCancellationRequested) { bool hasDetailProgress = false; foreach (var file in _fileUploadProgressLog) { string message; while (file.Value.TryDequeue(out message)) { hasDetailProgress = true; context.Output(message); } } // trace total file progress every 25 seconds when there is no file level detail progress if (++traceInterval % 2 == 0 && !hasDetailProgress) { context.Output(StringUtil.Loc("FileUploadProgress", totalFiles, _filesProcessed, (_filesProcessed * 100) / totalFiles)); } await Task.WhenAny(_uploadFinished.Task, Task.Delay(5000, token)); } }
private async Task ParallelUploadAsync(IAsyncCommandContext context, int uploaderId, CancellationToken token) { string fileToUpload; Stopwatch uploadTimer = new Stopwatch(); while (_fileUploadQueue.TryDequeue(out fileToUpload)) { token.ThrowIfCancellationRequested(); try { context.Output(StringUtil.Loc("StartFileUpload", fileToUpload, new FileInfo(fileToUpload).Length)); using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read)) { string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/'); uploadTimer.Restart(); HttpResponseMessage response = null; try { response = await _fileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, token); } catch (OperationCanceledException) when(token.IsCancellationRequested) { context.Output(StringUtil.Loc("FileUploadCancelled", fileToUpload)); throw; } catch (Exception ex) { context.Output(StringUtil.Loc("FileUploadFailed", fileToUpload, ex.Message)); throw; } uploadTimer.Stop(); if (response.StatusCode != System.Net.HttpStatusCode.Created) { throw new Exception(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, fileToUpload, itemPath)); } else { context.Output(StringUtil.Loc("FileUploadFinish", fileToUpload, uploadTimer.ElapsedMilliseconds)); } } Interlocked.Increment(ref _filesUploaded); } catch (Exception ex) when(!(ex is OperationCanceledException)) { _exceptionsDuringFileUpload.Add(ex); _uploadCancellationTokenSource.Cancel(); return; } } }
public async Task CopyToContainerAsync( IAsyncCommandContext context, String source, CancellationToken cancellationToken) { //set maxConcurrentUploads up to 2 untill figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2); //context.Output($"Max Concurrent Uploads {maxConcurrentUploads}"); IEnumerable <String> files; if (File.Exists(source)) { files = new List <String>() { source }; _sourceParentDirectory = Path.GetDirectoryName(source); } else { files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories); _sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); } context.Output(StringUtil.Loc("TotalUploadFiles", files.Count())); foreach (var file in files) { _fileUploadQueue.Enqueue(file); } using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { List <Task> allRunningTasks = new List <Task>(); // start reporting task to keep tracking upload progress. allRunningTasks.Add(ReportingAsync(context, files.Count(), _uploadCancellationTokenSource.Token)); // start parallel upload task. for (int i = 0; i < maxConcurrentUploads; i++) { allRunningTasks.Add(ParallelUploadAsync(context, i, _uploadCancellationTokenSource.Token)); } // the only expected type of exception will throw from both parallel upload task and reporting task is OperationCancelledException. try { await Task.WhenAll(allRunningTasks); } catch (OperationCanceledException) { // throw aggregate exception for all non-operationcancelexception we catched during file upload. if (_exceptionsDuringFileUpload.Count > 0) { throw new AggregateException(_exceptionsDuringFileUpload).Flatten(); } throw; } } }
private async Task ReportingAsync(IAsyncCommandContext context, int totalFiles, CancellationToken token) { while (_filesUploaded != totalFiles) { context.Output(StringUtil.Loc("FileUploadProgress", totalFiles, _filesUploaded)); await Task.Delay(2000, token); } }
private async Task AssociateReportingAsync(IAsyncCommandContext context, int totalFiles, TaskCompletionSource <int> uploadFinished, CancellationToken token) { while (!uploadFinished.Task.IsCompleted && !token.IsCancellationRequested) { context.Output(StringUtil.Loc("FileAssociateProgress", totalFiles, _filesProcessed, (_filesProcessed * 100) / totalFiles)); await Task.WhenAny(uploadFinished.Task, Task.Delay(10000, token)); } }
public async Task CopyToContainerAsync( IAsyncCommandContext context, String source, CancellationToken cancellationToken) { //set maxConcurrentUploads up to 2 untill figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2); //context.Output($"Max Concurrent Uploads {maxConcurrentUploads}"); IEnumerable<String> files; if (File.Exists(source)) { files = new List<String>() { source }; _sourceParentDirectory = Path.GetDirectoryName(source); } else { files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories); _sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); } context.Output(StringUtil.Loc("TotalUploadFiles", files.Count())); foreach (var file in files) { _fileUploadQueue.Enqueue(file); } using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { List<Task> allRunningTasks = new List<Task>(); // start reporting task to keep tracking upload progress. allRunningTasks.Add(ReportingAsync(context, files.Count(), _uploadCancellationTokenSource.Token)); // start parallel upload task. for (int i = 0; i < maxConcurrentUploads; i++) { allRunningTasks.Add(ParallelUploadAsync(context, i, _uploadCancellationTokenSource.Token)); } // the only expected type of exception will throw from both parallel upload task and reporting task is OperationCancelledException. try { await Task.WhenAll(allRunningTasks); } catch (OperationCanceledException) { // throw aggregate exception for all non-operationcancelexception we catched during file upload. if (_exceptionsDuringFileUpload.Count > 0) { throw new AggregateException(_exceptionsDuringFileUpload).Flatten(); } throw; } } }
private async Task UpdateBuildNumberAsync( IAsyncCommandContext context, VssConnection connection, Guid projectId, int buildId, string buildNumber, CancellationToken cancellationToken) { BuildServer buildServer = new BuildServer(connection, projectId); var build = await buildServer.UpdateBuildNumber(buildId, buildNumber, cancellationToken); context.Output(StringUtil.Loc("UpdateBuildNumberForBuild", build.BuildNumber, build.Id)); }
private async Task UpdateReleaseNameAsync( IAsyncCommandContext commandContext, IExecutionContext context, VssConnection connection, Guid projectId, string releaseId, string releaseName, CancellationToken cancellationToken) { ReleaseServer releaseServer = new ReleaseServer(connection, projectId); var release = await releaseServer.UpdateReleaseName(releaseId, releaseName, cancellationToken); commandContext.Output(StringUtil.Loc("RMUpdateReleaseNameForRelease", release.Name, release.Id)); context.Variables.Set("release.releaseName", release.Name); }
private async Task UpdateBuildNumberAsync( IAsyncCommandContext context, VssConnection connection, Guid projectId, int buildId, string buildNumber, CancellationToken cancellationToken) { var buildServer = context.GetHostContext().GetService <IBuildServer>(); await buildServer.ConnectAsync(connection); var build = await buildServer.UpdateBuildNumber(buildId, projectId, buildNumber, cancellationToken); context.Output(StringUtil.Loc("UpdateBuildNumberForBuild", build.BuildNumber, build.Id)); }
private async Task AssociateArtifactAsync( IAsyncCommandContext context, VssConnection connection, Guid projectId, int buildId, string name, string type, string data, Dictionary <string, string> propertiesDictionary, CancellationToken cancellationToken) { BuildServer buildHelper = new BuildServer(connection, projectId); var artifact = await buildHelper.AssociateArtifact(buildId, name, type, data, propertiesDictionary, cancellationToken); context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId)); }
public static async Task AssociateArtifactAsync( IAsyncCommandContext context, VssConnection connection, Guid projectId, int buildId, string name, string jobId, string type, string data, Dictionary <string, string> propertiesDictionary, CancellationToken cancellationToken) { var buildHelper = context.GetHostContext().GetService <IBuildServer>(); await buildHelper.ConnectAsync(connection); var artifact = await buildHelper.AssociateArtifactAsync(buildId, projectId, name, jobId, type, data, propertiesDictionary, cancellationToken); context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId)); }
private async Task AddBuildTagAsync( IAsyncCommandContext context, VssConnection connection, Guid projectId, int buildId, string buildTag, CancellationToken cancellationToken) { BuildServer buildServer = new BuildServer(connection, projectId); var tags = await buildServer.AddBuildTag(buildId, buildTag, cancellationToken); if (tags == null || !tags.Any(t => t.Equals(buildTag, StringComparison.OrdinalIgnoreCase))) { throw new Exception(StringUtil.Loc("BuildTagAddFailed", buildTag)); } else { context.Output(StringUtil.Loc("BuildTagsForBuild", buildId, String.Join(", ", tags))); } }
private async Task AddBuildTagAsync( IAsyncCommandContext context, Uri projectCollection, VssCredentials credentials, Guid projectId, int buildId, string buildTag, CancellationToken cancellationToken) { BuildServer buildServer = new BuildServer(projectCollection, credentials, projectId); var tags = await buildServer.AddBuildTag(buildId, buildTag, cancellationToken); if (tags == null || !tags.Contains(buildTag)) { throw new Exception(StringUtil.Loc("BuildTagAddFailed", buildTag)); } else { context.Output(StringUtil.Loc("BuildTagsForBuild", buildId, String.Join(", ", tags))); } }
public async Task PublishCodeCoverageFilesAsync(IAsyncCommandContext context, Guid projectId, long containerId, List <Tuple <string, string> > files, bool browsable, CancellationToken cancellationToken) { var publishCCTasks = files.Select(async file => { var browsableProperty = (browsable) ? bool.TrueString : bool.FalseString; var artifactProperties = new Dictionary <string, string> { { ArtifactUploadEventProperties.ContainerFolder, file.Item2 }, { ArtifactUploadEventProperties.ArtifactName, file.Item2 }, { ArtifactAssociateEventProperties.ArtifactType, ArtifactResourceTypes.Container }, { ArtifactAssociateEventProperties.Browsable, browsableProperty }, }; FileContainerServer fileContainerHelper = new FileContainerServer(_connection, projectId, containerId, file.Item2); await fileContainerHelper.CopyToContainerAsync(context, file.Item1, cancellationToken); string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{file.Item2}"); Build.BuildServer buildHelper = new Build.BuildServer(_connection, projectId); var artifact = await buildHelper.AssociateArtifact(_buildId, file.Item2, ArtifactResourceTypes.Container, fileContainerFullPath, artifactProperties, cancellationToken); context.Output(StringUtil.Loc("PublishedCodeCoverageArtifact", file.Item1, file.Item2)); }); await Task.WhenAll(publishCCTasks); }
public async Task PublishCodeCoverageFilesAsync(IAsyncCommandContext context, Guid projectId, long containerId, List<Tuple<string, string>> files, bool browsable, CancellationToken cancellationToken) { var publishCCTasks = files.Select(async file => { var browsableProperty = (browsable) ? bool.TrueString : bool.FalseString; var artifactProperties = new Dictionary<string, string> { { ArtifactUploadEventProperties.ContainerFolder, file.Item2}, { ArtifactUploadEventProperties.ArtifactName, file.Item2 }, { ArtifactAssociateEventProperties.ArtifactType, WellKnownArtifactResourceTypes.Container }, { ArtifactAssociateEventProperties.Browsable, browsableProperty }, }; FileContainerServer fileContainerHelper = new FileContainerServer(_connection.Uri, _connection.Credentials, projectId, containerId, file.Item2); await fileContainerHelper.CopyToContainerAsync(context, file.Item1, cancellationToken); string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{file.Item2}"); Build.BuildServer buildHelper = new Build.BuildServer(_connection.Uri, _connection.Credentials, projectId); var artifact = await buildHelper.AssociateArtifact(_buildId, file.Item2, WellKnownArtifactResourceTypes.Container, fileContainerFullPath, artifactProperties, cancellationToken); context.Output(StringUtil.Loc("PublishedCodeCoverageArtifact", file.Item1, file.Item2)); }); await Task.WhenAll(publishCCTasks); }
private async Task <List <string> > UploadAsync(IAsyncCommandContext context, int uploaderId, CancellationToken token) { List <string> failedFiles = new List <string>(); string fileToUpload; Stopwatch uploadTimer = new Stopwatch(); while (_fileUploadQueue.TryDequeue(out fileToUpload)) { token.ThrowIfCancellationRequested(); try { using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read, FileShare.Read)) { string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/'); uploadTimer.Restart(); bool catchExceptionDuringUpload = false; HttpResponseMessage response = null; try { response = await _fileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, cancellationToken : token, chunkSize : 4 * 1024 * 1024); } catch (OperationCanceledException) when(token.IsCancellationRequested) { context.Output(StringUtil.Loc("FileUploadCancelled", fileToUpload)); if (response != null) { response.Dispose(); response = null; } throw; } catch (Exception ex) { catchExceptionDuringUpload = true; context.Output(StringUtil.Loc("FileUploadFailed", fileToUpload, ex.Message)); context.Output(ex.ToString()); } uploadTimer.Stop(); if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created)) { if (response != null) { context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, fileToUpload, itemPath)); } // output detail upload trace for the file. ConcurrentQueue <string> logQueue; if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue)) { context.Output(StringUtil.Loc("FileUploadDetailTrace", itemPath)); string message; while (logQueue.TryDequeue(out message)) { context.Output(message); } } // tracking file that failed to upload. failedFiles.Add(fileToUpload); } else { context.Debug(StringUtil.Loc("FileUploadFinish", fileToUpload, uploadTimer.ElapsedMilliseconds)); // debug detail upload trace for the file. ConcurrentQueue <string> logQueue; if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue)) { context.Debug($"Detail upload trace for file: {itemPath}"); string message; while (logQueue.TryDequeue(out message)) { context.Debug(message); } } } if (response != null) { response.Dispose(); response = null; } } Interlocked.Increment(ref _filesProcessed); } catch (Exception ex) { context.Output(StringUtil.Loc("FileUploadFileOpenFailed", ex.Message, fileToUpload)); throw ex; } } return(failedFiles); }
private async Task ReportingAsync(IAsyncCommandContext context, int totalFiles, CancellationToken token) { while (_filesUploaded != totalFiles) { context.Output(StringUtil.Loc("FileUploadProgress", totalFiles, _filesUploaded)); await Task.Delay(2000, token); } }
private async Task <UploadResult> BlobUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token) { // return files that fail to upload and total artifact size var uploadResult = new UploadResult(); // nothing needs to upload if (files.Count == 0) { return(uploadResult); } var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase); var(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance .CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, token); // Upload to blobstore var results = await BlobStoreUtils.UploadBatchToBlobstore(verbose, files, (level, uri, type) => new BuildArtifactActionRecord(level, uri, type, nameof(BlobUploadAsync), context), (str) => context.Output(str), dedupClient, clientTelemetry, token, enableReporting : true); // Associate with TFS context.Output(StringUtil.Loc("AssociateFiles")); var queue = new ConcurrentQueue <BlobFileInfo>(); foreach (var file in results.fileDedupIds) { queue.Enqueue(file); } // Start associate monitor var uploadFinished = new TaskCompletionSource <int>(); var associateMonitor = AssociateReportingAsync(context, files.Count(), uploadFinished, token); // Start parallel associate tasks. var parallelAssociateTasks = new List <Task <UploadResult> >(); for (int uploader = 0; uploader < concurrentUploads; uploader++) { parallelAssociateTasks.Add(AssociateAsync(context, queue, token)); } // Wait for parallel associate tasks to finish. await Task.WhenAll(parallelAssociateTasks); foreach (var associateTask in parallelAssociateTasks) { // record all failed files. uploadResult.AddUploadResult(await associateTask); } // Stop monitor task uploadFinished.SetResult(0); await associateMonitor; // report telemetry if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.PlanId), out var planId)) { planId = Guid.Empty; } if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.JobId), out var jobId)) { jobId = Guid.Empty; } await clientTelemetry.CommitTelemetryUpload(planId, jobId); return(uploadResult); }
private async Task UpdateBuildNumberAsync( IAsyncCommandContext context, Uri projectCollection, VssCredentials credentials, Guid projectId, int buildId, string buildNumber, CancellationToken cancellationToken) { BuildServer buildServer = new BuildServer(projectCollection, credentials, projectId); var build = await buildServer.UpdateBuildNumber(buildId, buildNumber, cancellationToken); context.Output(StringUtil.Loc("UpdateBuildNumberForBuild", build.BuildNumber, build.Id)); }
private async Task ParallelUploadAsync(IAsyncCommandContext context, int uploaderId, CancellationToken token) { string fileToUpload; Stopwatch uploadTimer = new Stopwatch(); while (_fileUploadQueue.TryDequeue(out fileToUpload)) { token.ThrowIfCancellationRequested(); try { context.Output(StringUtil.Loc("StartFileUpload", fileToUpload, new FileInfo(fileToUpload).Length)); using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read)) { string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/'); uploadTimer.Restart(); var response = await FileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, token); uploadTimer.Stop(); if (response.StatusCode != System.Net.HttpStatusCode.Created) { throw new Exception(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, fileToUpload, itemPath)); } else { context.Output(StringUtil.Loc("FileUploadFinish", fileToUpload, uploadTimer.ElapsedMilliseconds)); } } Interlocked.Increment(ref _filesUploaded); } catch (Exception ex) when (!(ex is OperationCanceledException)) { _exceptionsDuringFileUpload.Add(ex); _uploadCancellationTokenSource.Cancel(); return; } } }
private async Task <UploadResult> AssociateAsync(IAsyncCommandContext context, ConcurrentQueue <BlobFileInfo> associateQueue, CancellationToken token) { var uploadResult = new UploadResult(); var retryHelper = new RetryHelper(context); var uploadTimer = new Stopwatch(); while (associateQueue.TryDequeue(out var file)) { uploadTimer.Restart(); string itemPath = (_containerPath.TrimEnd('/') + "/" + file.Path.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/'); bool catchExceptionDuringUpload = false; HttpResponseMessage response = null; try { if (file.Success) { var length = (long)file.Node.TransitiveContentBytes; response = await retryHelper.Retry(async() => await _fileContainerHttpClient.CreateItemForArtifactUpload(_containerId, itemPath, _projectId, file.DedupId.ValueString, length, token), (retryCounter) => (int)Math.Pow(retryCounter, 2) * 5, (exception) => true); uploadResult.TotalFileSizeUploaded += length; } } catch (OperationCanceledException) when(token.IsCancellationRequested) { context.Output(StringUtil.Loc("FileUploadCancelled", itemPath)); if (response != null) { response.Dispose(); } throw; } catch (Exception ex) { catchExceptionDuringUpload = true; context.Output(StringUtil.Loc("FileUploadFailed", itemPath, ex.Message)); context.Output(ex.ToString()); } if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created) || !file.Success) { if (response != null) { context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, file.Path, itemPath)); } if (!file.Success) { context.Output(StringUtil.Loc("FileContainerUploadFailedBlob", file.Path, itemPath)); } // tracking file that failed to upload. uploadResult.FailedFiles.Add(file.Path); } else { context.Debug(StringUtil.Loc("FileUploadFinish", file.Path, uploadTimer.ElapsedMilliseconds)); } if (response != null) { response.Dispose(); } Interlocked.Increment(ref _filesProcessed); } return(uploadResult); }
private async Task PublishCodeCoverageAsync(IExecutionContext executionContext, IAsyncCommandContext commandContext, ICodeCoveragePublisher codeCoveragePublisher, IEnumerable<CodeCoverageStatistics> coverageData, string project, Guid projectId, long containerId, CancellationToken cancellationToken) { //step 2: publish code coverage summary to TFS if (coverageData != null && coverageData.Count() > 0) { commandContext.Output(StringUtil.Loc("PublishingCodeCoverage")); foreach (var coverage in coverageData) { commandContext.Output(StringUtil.Format(" {0}- {1} of {2} covered.", coverage.Label, coverage.Covered, coverage.Total)); } await codeCoveragePublisher.PublishCodeCoverageSummaryAsync(coverageData, project, cancellationToken); } // step 3: publish code coverage files as build artifacts string additionalCodeCoverageFilePath = null; string destinationSummaryFile = null; var newReportDirectory = _reportDirectory; try { var filesToPublish = new List<Tuple<string, string>>(); if (!Directory.Exists(newReportDirectory)) { if (!string.IsNullOrWhiteSpace(newReportDirectory)) { // user gave a invalid report directory. Write warning and continue. executionContext.Warning(StringUtil.Loc("DirectoryNotFound", newReportDirectory)); } newReportDirectory = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.ReportDirectory); Directory.CreateDirectory(newReportDirectory); } var summaryFileName = Path.GetFileName(_summaryFileLocation); destinationSummaryFile = Path.Combine(newReportDirectory, CodeCoverageConstants.SummaryFileDirectory + _buildId, summaryFileName); Directory.CreateDirectory(Path.GetDirectoryName(destinationSummaryFile)); File.Copy(_summaryFileLocation, destinationSummaryFile, true); filesToPublish.Add(new Tuple<string, string>(newReportDirectory, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.ReportDirectory))); if (_additionalCodeCoverageFiles != null && _additionalCodeCoverageFiles.Count != 0) { additionalCodeCoverageFilePath = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory); CodeCoverageUtilities.CopyFilesFromFileListWithDirStructure(_additionalCodeCoverageFiles, ref additionalCodeCoverageFilePath); filesToPublish.Add(new Tuple<string, string>(additionalCodeCoverageFilePath, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory))); } commandContext.Output(StringUtil.Loc("PublishingCodeCoverageFiles")); await codeCoveragePublisher.PublishCodeCoverageFilesAsync(commandContext, projectId, containerId, filesToPublish, File.Exists(Path.Combine(newReportDirectory, CodeCoverageConstants.DefaultIndexFile)), cancellationToken); } catch (IOException ex) { executionContext.Warning(StringUtil.Loc("ErrorOccuredWhilePublishingCCFiles", ex.Message)); } finally { // clean temporary files. if (!string.IsNullOrEmpty(additionalCodeCoverageFilePath)) { if (Directory.Exists(additionalCodeCoverageFilePath)) { Directory.Delete(path: additionalCodeCoverageFilePath, recursive: true); } } if (!string.IsNullOrEmpty(destinationSummaryFile)) { var summaryFileDirectory = Path.GetDirectoryName(destinationSummaryFile); if (Directory.Exists(summaryFileDirectory)) { Directory.Delete(path: summaryFileDirectory, recursive: true); } } if (!Directory.Exists(_reportDirectory)) { if (Directory.Exists(newReportDirectory)) { //delete the generated report directory Directory.Delete(path: newReportDirectory, recursive: true); } } } }
private async Task ProcessPluginCommandAsync(IAsyncCommandContext context, AgentCommandPluginExecutionContext pluginContext, string plugin, Command command, CancellationToken token) { // Resolve the working directory. string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); ArgUtil.Directory(workingDirectory, nameof(workingDirectory)); // Agent.PluginHost string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}"); // Agent.PluginHost's arguments string arguments = $"command \"{plugin}\""; // Execute the process. Exit code 0 should always be returned. // A non-zero exit code indicates infrastructural failure. // Any content coming from STDERR will indicate the command process failed. // We can't use ## command for plugin to communicate, since we are already processing ## command using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { object stderrLock = new object(); List <string> stderr = new List <string>(); processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => { context.Output(e.Data); }; processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => { lock (stderrLock) { stderr.Add(e.Data); }; }; var redirectStandardIn = new InputQueue <string>(); redirectStandardIn.Enqueue(JsonUtility.ToString(pluginContext)); int returnCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory, fileName : file, arguments : arguments, environment : null, requireExitCodeZero : false, outputEncoding : null, killProcessOnCancel : false, redirectStandardIn : redirectStandardIn, cancellationToken : token); if (returnCode != 0) { context.Output(string.Join(Environment.NewLine, stderr)); throw new ProcessExitCodeException(returnCode, file, arguments); } else if (stderr.Count > 0) { throw new InvalidOperationException(string.Join(Environment.NewLine, stderr)); } else { // Everything works fine. // Return code is 0. // No STDERR comes out. } } }
private async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(IAsyncCommandContext context, string itemPath, CancellationToken cancellationToken) { // Create chunks and identifier var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false); var rootNode = new DedupNode(new [] { chunk }); var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K); // Setup upload session to keep file for at mimimum one day var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase); var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, (str) => context.Output(str)); var keepUntulRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1)); var uploadSession = _dedupClient.CreateUploadSession(keepUntulRef, tracer, FileSystem.Instance); // Upload the chunks var uploadRecord = _blobTelemetry.CreateRecord <BuildArtifactActionRecord>((level, uri, type) => new BuildArtifactActionRecord(level, uri, type, nameof(UploadAsync), context)); await _blobTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync( async() => { return(await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>() { [dedupId] = itemPath }, cancellationToken)); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times cancellationToken: cancellationToken, continueOnCapturedContext: false) ); return(dedupId, rootNode.TransitiveContentBytes); }
public RetryHelper(IAsyncCommandContext commandContext, int maxRetries = 3) { Debug = (str) => commandContext.Debug(str); Warning = (str) => commandContext.Output(str); MaxRetries = maxRetries; }
private async Task <UploadResult> BlobUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token) { // return files that fail to upload and total artifact size var uploadResult = new UploadResult(); // nothing needs to upload if (files.Count == 0) { return(uploadResult); } DedupStoreClient dedupClient = null; BlobStoreClientTelemetryTfs clientTelemetry = null; try { var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase); int maxParallelism = context.GetHostContext().GetService <IConfigurationStore>().GetSettings().MaxDedupParallelism; (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance .CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, maxParallelism, token); // Upload to blobstore var results = await BlobStoreUtils.UploadBatchToBlobstore(verbose, files, (level, uri, type) => new BuildArtifactActionRecord(level, uri, type, nameof(BlobUploadAsync), context), (str) => context.Output(str), dedupClient, clientTelemetry, token, enableReporting : true); // Associate with TFS context.Output(StringUtil.Loc("AssociateFiles")); var queue = new ConcurrentQueue <BlobFileInfo>(); foreach (var file in results.fileDedupIds) { queue.Enqueue(file); } // Start associate monitor var uploadFinished = new TaskCompletionSource <int>(); var associateMonitor = AssociateReportingAsync(context, files.Count(), uploadFinished, token); // Start parallel associate tasks. var parallelAssociateTasks = new List <Task <UploadResult> >(); for (int uploader = 0; uploader < concurrentUploads; uploader++) { parallelAssociateTasks.Add(AssociateAsync(context, queue, token)); } // Wait for parallel associate tasks to finish. await Task.WhenAll(parallelAssociateTasks); foreach (var associateTask in parallelAssociateTasks) { // record all failed files. uploadResult.AddUploadResult(await associateTask); } // Stop monitor task uploadFinished.SetResult(0); await associateMonitor; // report telemetry if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.PlanId), out var planId)) { planId = Guid.Empty; } if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.JobId), out var jobId)) { jobId = Guid.Empty; } await clientTelemetry.CommitTelemetryUpload(planId, jobId); } catch (SocketException e) { ExceptionsUtil.HandleSocketException(e, this._connection.Uri.ToString(), context.Warn); throw; } catch { var blobStoreHost = dedupClient.Client.BaseAddress.Host; var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform(); var warningMessage = StringUtil.Loc("BlobStoreUploadWarning", blobStoreHost, allowListLink); context.Warn(warningMessage); throw; } return(uploadResult); }
private async Task UploadArtifactAsync( IAsyncCommandContext context, Uri projectCollection, VssCredentials credentials, Guid projectId, long containerId, string containerPath, int buildId, string name, Dictionary<string, string> propertiesDictionary, string source, CancellationToken cancellationToken) { FileContainerServer fileContainerHelper = new FileContainerServer(projectCollection, credentials, projectId, containerId, containerPath); await fileContainerHelper.CopyToContainerAsync(context, source, cancellationToken); string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{containerPath}"); context.Output(StringUtil.Loc("UploadToFileContainer", source, fileContainerFullPath)); BuildServer buildHelper = new BuildServer(projectCollection, credentials, projectId); var artifact = await buildHelper.AssociateArtifact(buildId, name, WellKnownArtifactResourceTypes.Container, fileContainerFullPath, propertiesDictionary, cancellationToken); context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId)); }
private async Task AssociateArtifactAsync( IAsyncCommandContext context, Uri projectCollection, VssCredentials credentials, Guid projectId, int buildId, string name, string type, string data, Dictionary<string, string> propertiesDictionary, CancellationToken cancellationToken) { BuildServer buildHelper = new BuildServer(projectCollection, credentials, projectId); var artifact = await buildHelper.AssociateArtifact(buildId, name, type, data, propertiesDictionary, cancellationToken); context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId)); }
public async Task CopyToContainerAsync( IAsyncCommandContext context, String source, CancellationToken cancellationToken) { //set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2); //context.Output($"Max Concurrent Uploads {maxConcurrentUploads}"); List <String> files; if (File.Exists(source)) { files = new List <String>() { source }; _sourceParentDirectory = Path.GetDirectoryName(source); } else { files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList(); _sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); } context.Output(StringUtil.Loc("TotalUploadFiles", files.Count())); using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { // hook up reporting event from file container client. _fileContainerHttpClient.UploadFileReportTrace += UploadFileTraceReportReceived; _fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived; try { // try upload all files for the first time. List <string> failedFiles = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token); if (failedFiles.Count == 0) { // all files have been upload succeed. context.Output(StringUtil.Loc("FileUploadSucceed")); return; } else { context.Output(StringUtil.Loc("FileUploadFailedRetryLater", failedFiles.Count)); } // Delay 1 min then retry failed files. for (int timer = 60; timer > 0; timer -= 5) { context.Output(StringUtil.Loc("FileUploadRetryInSecond", timer)); await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token); } // Retry upload all failed files. context.Output(StringUtil.Loc("FileUploadRetry", failedFiles.Count)); failedFiles = await ParallelUploadAsync(context, failedFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token); if (failedFiles.Count == 0) { // all files have been upload succeed after retry. context.Output(StringUtil.Loc("FileUploadRetrySucceed")); return; } else { throw new Exception(StringUtil.Loc("FileUploadFailedAfterRetry")); } } finally { _fileContainerHttpClient.UploadFileReportTrace -= UploadFileTraceReportReceived; _fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived; } } }
private async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(IAsyncCommandContext context, string itemPath, CancellationToken cancellationToken) { var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase); return(await BlobStoreUtils.UploadToBlobStore(verbose, itemPath, (level, uri, type) => new BuildArtifactActionRecord(level, uri, type, nameof(UploadToBlobStore), context), (str) => context.Output(str), _dedupClient, _blobTelemetry, cancellationToken)); }
private async Task PublishCodeCoverageAsync(IExecutionContext executionContext, IAsyncCommandContext commandContext, ICodeCoveragePublisher codeCoveragePublisher, IEnumerable <CodeCoverageStatistics> coverageData, string project, Guid projectId, long containerId, CancellationToken cancellationToken) { //step 2: publish code coverage summary to TFS if (coverageData != null && coverageData.Count() > 0) { commandContext.Output(StringUtil.Loc("PublishingCodeCoverage")); foreach (var coverage in coverageData) { commandContext.Output(StringUtil.Format(" {0}- {1} of {2} covered.", coverage.Label, coverage.Covered, coverage.Total)); } await codeCoveragePublisher.PublishCodeCoverageSummaryAsync(coverageData, project, cancellationToken); } // step 3: publish code coverage files as build artifacts string additionalCodeCoverageFilePath = null; string destinationSummaryFile = null; var newReportDirectory = _reportDirectory; try { var filesToPublish = new List <Tuple <string, string> >(); if (!Directory.Exists(newReportDirectory)) { if (!string.IsNullOrWhiteSpace(newReportDirectory)) { // user gave a invalid report directory. Write warning and continue. executionContext.Warning(StringUtil.Loc("DirectoryNotFound", newReportDirectory)); } newReportDirectory = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.ReportDirectory); Directory.CreateDirectory(newReportDirectory); } var summaryFileName = Path.GetFileName(_summaryFileLocation); destinationSummaryFile = Path.Combine(newReportDirectory, CodeCoverageConstants.SummaryFileDirectory + _buildId, summaryFileName); Directory.CreateDirectory(Path.GetDirectoryName(destinationSummaryFile)); File.Copy(_summaryFileLocation, destinationSummaryFile, true); filesToPublish.Add(new Tuple <string, string>(newReportDirectory, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.ReportDirectory))); if (_additionalCodeCoverageFiles != null && _additionalCodeCoverageFiles.Count != 0) { additionalCodeCoverageFilePath = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory); CodeCoverageUtilities.CopyFilesFromFileListWithDirStructure(_additionalCodeCoverageFiles, ref additionalCodeCoverageFilePath); filesToPublish.Add(new Tuple <string, string>(additionalCodeCoverageFilePath, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory))); } commandContext.Output(StringUtil.Loc("PublishingCodeCoverageFiles")); await codeCoveragePublisher.PublishCodeCoverageFilesAsync(commandContext, projectId, containerId, filesToPublish, File.Exists(Path.Combine(newReportDirectory, CodeCoverageConstants.DefaultIndexFile)), cancellationToken); } catch (IOException ex) { executionContext.Warning(StringUtil.Loc("ErrorOccuredWhilePublishingCCFiles", ex.Message)); } finally { // clean temporary files. if (!string.IsNullOrEmpty(additionalCodeCoverageFilePath)) { if (Directory.Exists(additionalCodeCoverageFilePath)) { Directory.Delete(path: additionalCodeCoverageFilePath, recursive: true); } } if (!string.IsNullOrEmpty(destinationSummaryFile)) { var summaryFileDirectory = Path.GetDirectoryName(destinationSummaryFile); if (Directory.Exists(summaryFileDirectory)) { Directory.Delete(path: summaryFileDirectory, recursive: true); } } if (!Directory.Exists(_reportDirectory)) { if (Directory.Exists(newReportDirectory)) { //delete the generated report directory Directory.Delete(path: newReportDirectory, recursive: true); } } } }
private async Task AddBuildTagAsync( IAsyncCommandContext context, Uri projectCollection, VssCredentials credentials, Guid projectId, int buildId, string buildTag, CancellationToken cancellationToken) { BuildServer buildServer = new BuildServer(projectCollection, credentials, projectId); var tags = await buildServer.AddBuildTag(buildId, buildTag, cancellationToken); if (tags == null || !tags.Contains(buildTag)) { throw new Exception(StringUtil.Loc("BuildTagAddFailed", buildTag)); } else { context.Output(StringUtil.Loc("BuildTagsForBuild", buildId, String.Join(", ", tags))); } }
private async Task <UploadResult> ParallelUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token) { // return files that fail to upload and total artifact size var uploadResult = new UploadResult(); // nothing needs to upload if (files.Count == 0) { return(uploadResult); } var uploadToBlob = String.Equals(context.GetVariableValueOrDefault("agent.UploadBuildArtifactsToBlob"), "true", StringComparison.InvariantCultureIgnoreCase); if (uploadToBlob) { var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase); var(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance .CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, token); _dedupClient = dedupClient; _blobTelemetry = clientTelemetry; } // ensure the file upload queue is empty. if (!_fileUploadQueue.IsEmpty) { throw new ArgumentOutOfRangeException(nameof(_fileUploadQueue)); } // enqueue file into upload queue. foreach (var file in files) { _fileUploadQueue.Enqueue(file); } // Start upload monitor task. _filesProcessed = 0; _uploadFinished = new TaskCompletionSource <int>(); _fileUploadTraceLog.Clear(); _fileUploadProgressLog.Clear(); Task uploadMonitor = ReportingAsync(context, files.Count(), _uploadCancellationTokenSource.Token); // Start parallel upload tasks. List <Task <UploadResult> > parallelUploadingTasks = new List <Task <UploadResult> >(); for (int uploader = 0; uploader < concurrentUploads; uploader++) { parallelUploadingTasks.Add(UploadAsync(context, uploader, uploadToBlob, _uploadCancellationTokenSource.Token)); } // Wait for parallel upload finish. await Task.WhenAll(parallelUploadingTasks); foreach (var uploadTask in parallelUploadingTasks) { // record all failed files. uploadResult.AddUploadResult(await uploadTask); } // Stop monitor task; _uploadFinished.TrySetResult(0); await uploadMonitor; // report telemetry if (uploadToBlob) { if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.PlanId), out var planId)) { planId = Guid.Empty; } if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.JobId), out var jobId)) { jobId = Guid.Empty; } await _blobTelemetry.CommitTelemetry(planId, jobId); } return(uploadResult); }
public async Task <long> CopyToContainerAsync( IAsyncCommandContext context, String source, CancellationToken cancellationToken) { ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(source, nameof(source)); //set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2); //context.Output($"Max Concurrent Uploads {maxConcurrentUploads}"); List <String> files; if (File.Exists(source)) { files = new List <String>() { source }; _sourceParentDirectory = Path.GetDirectoryName(source); } else { files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList(); _sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); } context.Output(StringUtil.Loc("TotalUploadFiles", files.Count())); using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { // hook up reporting event from file container client. _fileContainerHttpClient.UploadFileReportTrace += UploadFileTraceReportReceived; _fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived; try { var uploadToBlob = String.Equals(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.UploadBuildArtifactsToBlob), "true", StringComparison.InvariantCultureIgnoreCase) && !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean(); // try upload all files for the first time. UploadResult uploadResult = null; if (uploadToBlob) { try { uploadResult = await BlobUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token); } catch { // Fall back to FCS upload if we cannot upload to blob context.Warn(StringUtil.Loc("BlobStoreUploadWarning")); uploadToBlob = false; } } if (!uploadToBlob) { uploadResult = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token); } if (uploadResult.FailedFiles.Count == 0) { // all files have been upload succeed. context.Output(StringUtil.Loc("FileUploadSucceed")); return(uploadResult.TotalFileSizeUploaded); } else { context.Output(StringUtil.Loc("FileUploadFailedRetryLater", uploadResult.FailedFiles.Count)); } // Delay 1 min then retry failed files. for (int timer = 60; timer > 0; timer -= 5) { context.Output(StringUtil.Loc("FileUploadRetryInSecond", timer)); await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token); } // Retry upload all failed files. context.Output(StringUtil.Loc("FileUploadRetry", uploadResult.FailedFiles.Count)); UploadResult retryUploadResult; if (uploadToBlob) { retryUploadResult = await BlobUploadAsync(context, uploadResult.FailedFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token); } else { retryUploadResult = await ParallelUploadAsync(context, uploadResult.FailedFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token); } if (retryUploadResult.FailedFiles.Count == 0) { // all files have been upload succeed after retry. context.Output(StringUtil.Loc("FileUploadRetrySucceed")); return(uploadResult.TotalFileSizeUploaded + retryUploadResult.TotalFileSizeUploaded); } else { throw new Exception(StringUtil.Loc("FileUploadFailedAfterRetry")); } } finally { _fileContainerHttpClient.UploadFileReportTrace -= UploadFileTraceReportReceived; _fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived; } } }