public static async Task UploadArtifactAsync(
IAsyncCommandContext context,
VssConnection connection,
Guid projectId,
long containerId,
string containerPath,
int buildId,
string name,
string jobId,
Dictionary <string, string> propertiesDictionary,
string source,
CancellationToken cancellationToken)
{
var fileContainerHelper = new FileContainerServer(connection, projectId, containerId, containerPath);
var size = await fileContainerHelper.CopyToContainerAsync(context, source, cancellationToken);
propertiesDictionary.Add(ArtifactUploadEventProperties.ArtifactSize, size.ToString());
var fileContainerFullPath = StringUtil.Format($"#/{containerId}/{containerPath}");
context.Output(StringUtil.Loc("UploadToFileContainer", source, fileContainerFullPath));
var buildHelper = context.GetHostContext().GetService <IBuildServer>();
await buildHelper.ConnectAsync(connection);
var artifact = await buildHelper.AssociateArtifactAsync(buildId, projectId, name, jobId, ArtifactResourceTypes.Container, fileContainerFullPath, propertiesDictionary, cancellationToken);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId));
}
private async Task UploadArtifactAsync(
IAsyncCommandContext context,
Uri projectCollection,
VssCredentials credentials,
Guid projectId,
long containerId,
string containerPath,
int buildId,
string name,
Dictionary <string, string> propertiesDictionary,
string source,
CancellationToken cancellationToken)
{
FileContainerServer fileContainerHelper = new FileContainerServer(projectCollection, credentials, projectId, containerId, containerPath);
await fileContainerHelper.CopyToContainerAsync(context, source, cancellationToken);
string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{containerPath}");
context.Output(StringUtil.Loc("UploadToFileContainer", source, fileContainerFullPath));
BuildServer buildHelper = new BuildServer(projectCollection, credentials, projectId);
var artifact = await buildHelper.AssociateArtifact(buildId, name, WellKnownArtifactResourceTypes.Container, fileContainerFullPath, propertiesDictionary, cancellationToken);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId));
}
private async Task ReportingAsync(IAsyncCommandContext context, int totalFiles, CancellationToken token)
{
int traceInterval = 0;
while (!_uploadFinished.Task.IsCompleted && !token.IsCancellationRequested)
{
bool hasDetailProgress = false;
foreach (var file in _fileUploadProgressLog)
{
string message;
while (file.Value.TryDequeue(out message))
{
hasDetailProgress = true;
context.Output(message);
}
}
// trace total file progress every 25 seconds when there is no file level detail progress
if (++traceInterval % 2 == 0 && !hasDetailProgress)
{
context.Output(StringUtil.Loc("FileUploadProgress", totalFiles, _filesProcessed, (_filesProcessed * 100) / totalFiles));
}
await Task.WhenAny(_uploadFinished.Task, Task.Delay(5000, token));
}
}
private async Task ParallelUploadAsync(IAsyncCommandContext context, int uploaderId, CancellationToken token)
{
string fileToUpload;
Stopwatch uploadTimer = new Stopwatch();
while (_fileUploadQueue.TryDequeue(out fileToUpload))
{
token.ThrowIfCancellationRequested();
try
{
context.Output(StringUtil.Loc("StartFileUpload", fileToUpload, new FileInfo(fileToUpload).Length));
using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read))
{
string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
uploadTimer.Restart();
HttpResponseMessage response = null;
try
{
response = await _fileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, token);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Output(StringUtil.Loc("FileUploadCancelled", fileToUpload));
throw;
}
catch (Exception ex)
{
context.Output(StringUtil.Loc("FileUploadFailed", fileToUpload, ex.Message));
throw;
}
uploadTimer.Stop();
if (response.StatusCode != System.Net.HttpStatusCode.Created)
{
throw new Exception(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, fileToUpload, itemPath));
}
else
{
context.Output(StringUtil.Loc("FileUploadFinish", fileToUpload, uploadTimer.ElapsedMilliseconds));
}
}
Interlocked.Increment(ref _filesUploaded);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
_exceptionsDuringFileUpload.Add(ex);
_uploadCancellationTokenSource.Cancel();
return;
}
}
}
public async Task CopyToContainerAsync(
IAsyncCommandContext context,
String source,
CancellationToken cancellationToken)
{
//set maxConcurrentUploads up to 2 untill figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit
int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2);
//context.Output($"Max Concurrent Uploads {maxConcurrentUploads}");
IEnumerable <String> files;
if (File.Exists(source))
{
files = new List <String>()
{
source
};
_sourceParentDirectory = Path.GetDirectoryName(source);
}
else
{
files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories);
_sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
context.Output(StringUtil.Loc("TotalUploadFiles", files.Count()));
foreach (var file in files)
{
_fileUploadQueue.Enqueue(file);
}
using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
List <Task> allRunningTasks = new List <Task>();
// start reporting task to keep tracking upload progress.
allRunningTasks.Add(ReportingAsync(context, files.Count(), _uploadCancellationTokenSource.Token));
// start parallel upload task.
for (int i = 0; i < maxConcurrentUploads; i++)
{
allRunningTasks.Add(ParallelUploadAsync(context, i, _uploadCancellationTokenSource.Token));
}
// the only expected type of exception will throw from both parallel upload task and reporting task is OperationCancelledException.
try
{
await Task.WhenAll(allRunningTasks);
}
catch (OperationCanceledException)
{
// throw aggregate exception for all non-operationcancelexception we catched during file upload.
if (_exceptionsDuringFileUpload.Count > 0)
{
throw new AggregateException(_exceptionsDuringFileUpload).Flatten();
}
throw;
}
}
}
private async Task ReportingAsync(IAsyncCommandContext context, int totalFiles, CancellationToken token)
{
while (_filesUploaded != totalFiles)
{
context.Output(StringUtil.Loc("FileUploadProgress", totalFiles, _filesUploaded));
await Task.Delay(2000, token);
}
}
private async Task AssociateReportingAsync(IAsyncCommandContext context, int totalFiles, TaskCompletionSource <int> uploadFinished, CancellationToken token)
{
while (!uploadFinished.Task.IsCompleted && !token.IsCancellationRequested)
{
context.Output(StringUtil.Loc("FileAssociateProgress", totalFiles, _filesProcessed, (_filesProcessed * 100) / totalFiles));
await Task.WhenAny(uploadFinished.Task, Task.Delay(10000, token));
}
}
public async Task CopyToContainerAsync(
IAsyncCommandContext context,
String source,
CancellationToken cancellationToken)
{
//set maxConcurrentUploads up to 2 untill figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit
int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2);
//context.Output($"Max Concurrent Uploads {maxConcurrentUploads}");
IEnumerable<String> files;
if (File.Exists(source))
{
files = new List<String>() { source };
_sourceParentDirectory = Path.GetDirectoryName(source);
}
else
{
files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories);
_sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
context.Output(StringUtil.Loc("TotalUploadFiles", files.Count()));
foreach (var file in files)
{
_fileUploadQueue.Enqueue(file);
}
using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
List<Task> allRunningTasks = new List<Task>();
// start reporting task to keep tracking upload progress.
allRunningTasks.Add(ReportingAsync(context, files.Count(), _uploadCancellationTokenSource.Token));
// start parallel upload task.
for (int i = 0; i < maxConcurrentUploads; i++)
{
allRunningTasks.Add(ParallelUploadAsync(context, i, _uploadCancellationTokenSource.Token));
}
// the only expected type of exception will throw from both parallel upload task and reporting task is OperationCancelledException.
try
{
await Task.WhenAll(allRunningTasks);
}
catch (OperationCanceledException)
{
// throw aggregate exception for all non-operationcancelexception we catched during file upload.
if (_exceptionsDuringFileUpload.Count > 0)
{
throw new AggregateException(_exceptionsDuringFileUpload).Flatten();
}
throw;
}
}
}
private async Task UpdateBuildNumberAsync(
IAsyncCommandContext context,
VssConnection connection,
Guid projectId,
int buildId,
string buildNumber,
CancellationToken cancellationToken)
{
BuildServer buildServer = new BuildServer(connection, projectId);
var build = await buildServer.UpdateBuildNumber(buildId, buildNumber, cancellationToken);
context.Output(StringUtil.Loc("UpdateBuildNumberForBuild", build.BuildNumber, build.Id));
}
private async Task UpdateReleaseNameAsync(
IAsyncCommandContext commandContext,
IExecutionContext context,
VssConnection connection,
Guid projectId,
string releaseId,
string releaseName,
CancellationToken cancellationToken)
{
ReleaseServer releaseServer = new ReleaseServer(connection, projectId);
var release = await releaseServer.UpdateReleaseName(releaseId, releaseName, cancellationToken);
commandContext.Output(StringUtil.Loc("RMUpdateReleaseNameForRelease", release.Name, release.Id));
context.Variables.Set("release.releaseName", release.Name);
}
private async Task UpdateBuildNumberAsync(
IAsyncCommandContext context,
VssConnection connection,
Guid projectId,
int buildId,
string buildNumber,
CancellationToken cancellationToken)
{
var buildServer = context.GetHostContext().GetService <IBuildServer>();
await buildServer.ConnectAsync(connection);
var build = await buildServer.UpdateBuildNumber(buildId, projectId, buildNumber, cancellationToken);
context.Output(StringUtil.Loc("UpdateBuildNumberForBuild", build.BuildNumber, build.Id));
}
private async Task AssociateArtifactAsync(
IAsyncCommandContext context,
VssConnection connection,
Guid projectId,
int buildId,
string name,
string type,
string data,
Dictionary <string, string> propertiesDictionary,
CancellationToken cancellationToken)
{
BuildServer buildHelper = new BuildServer(connection, projectId);
var artifact = await buildHelper.AssociateArtifact(buildId, name, type, data, propertiesDictionary, cancellationToken);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId));
}
public static async Task AssociateArtifactAsync(
IAsyncCommandContext context,
VssConnection connection,
Guid projectId,
int buildId,
string name,
string jobId,
string type,
string data,
Dictionary <string, string> propertiesDictionary,
CancellationToken cancellationToken)
{
var buildHelper = context.GetHostContext().GetService <IBuildServer>();
await buildHelper.ConnectAsync(connection);
var artifact = await buildHelper.AssociateArtifactAsync(buildId, projectId, name, jobId, type, data, propertiesDictionary, cancellationToken);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId));
}
private async Task AddBuildTagAsync(
IAsyncCommandContext context,
VssConnection connection,
Guid projectId,
int buildId,
string buildTag,
CancellationToken cancellationToken)
{
BuildServer buildServer = new BuildServer(connection, projectId);
var tags = await buildServer.AddBuildTag(buildId, buildTag, cancellationToken);
if (tags == null || !tags.Any(t => t.Equals(buildTag, StringComparison.OrdinalIgnoreCase)))
{
throw new Exception(StringUtil.Loc("BuildTagAddFailed", buildTag));
}
else
{
context.Output(StringUtil.Loc("BuildTagsForBuild", buildId, String.Join(", ", tags)));
}
}
private async Task AddBuildTagAsync(
IAsyncCommandContext context,
Uri projectCollection,
VssCredentials credentials,
Guid projectId,
int buildId,
string buildTag,
CancellationToken cancellationToken)
{
BuildServer buildServer = new BuildServer(projectCollection, credentials, projectId);
var tags = await buildServer.AddBuildTag(buildId, buildTag, cancellationToken);
if (tags == null || !tags.Contains(buildTag))
{
throw new Exception(StringUtil.Loc("BuildTagAddFailed", buildTag));
}
else
{
context.Output(StringUtil.Loc("BuildTagsForBuild", buildId, String.Join(", ", tags)));
}
}
public async Task PublishCodeCoverageFilesAsync(IAsyncCommandContext context, Guid projectId, long containerId, List <Tuple <string, string> > files, bool browsable, CancellationToken cancellationToken)
{
var publishCCTasks = files.Select(async file =>
{
var browsableProperty = (browsable) ? bool.TrueString : bool.FalseString;
var artifactProperties = new Dictionary <string, string> {
{ ArtifactUploadEventProperties.ContainerFolder, file.Item2 },
{ ArtifactUploadEventProperties.ArtifactName, file.Item2 },
{ ArtifactAssociateEventProperties.ArtifactType, ArtifactResourceTypes.Container },
{ ArtifactAssociateEventProperties.Browsable, browsableProperty },
};
FileContainerServer fileContainerHelper = new FileContainerServer(_connection, projectId, containerId, file.Item2);
await fileContainerHelper.CopyToContainerAsync(context, file.Item1, cancellationToken);
string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{file.Item2}");
Build.BuildServer buildHelper = new Build.BuildServer(_connection, projectId);
var artifact = await buildHelper.AssociateArtifact(_buildId, file.Item2, ArtifactResourceTypes.Container, fileContainerFullPath, artifactProperties, cancellationToken);
context.Output(StringUtil.Loc("PublishedCodeCoverageArtifact", file.Item1, file.Item2));
});
await Task.WhenAll(publishCCTasks);
}
public async Task PublishCodeCoverageFilesAsync(IAsyncCommandContext context, Guid projectId, long containerId, List<Tuple<string, string>> files, bool browsable, CancellationToken cancellationToken)
{
var publishCCTasks = files.Select(async file =>
{
var browsableProperty = (browsable) ? bool.TrueString : bool.FalseString;
var artifactProperties = new Dictionary<string, string> {
{ ArtifactUploadEventProperties.ContainerFolder, file.Item2},
{ ArtifactUploadEventProperties.ArtifactName, file.Item2 },
{ ArtifactAssociateEventProperties.ArtifactType, WellKnownArtifactResourceTypes.Container },
{ ArtifactAssociateEventProperties.Browsable, browsableProperty },
};
FileContainerServer fileContainerHelper = new FileContainerServer(_connection.Uri, _connection.Credentials, projectId, containerId, file.Item2);
await fileContainerHelper.CopyToContainerAsync(context, file.Item1, cancellationToken);
string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{file.Item2}");
Build.BuildServer buildHelper = new Build.BuildServer(_connection.Uri, _connection.Credentials, projectId);
var artifact = await buildHelper.AssociateArtifact(_buildId, file.Item2, WellKnownArtifactResourceTypes.Container, fileContainerFullPath, artifactProperties, cancellationToken);
context.Output(StringUtil.Loc("PublishedCodeCoverageArtifact", file.Item1, file.Item2));
});
await Task.WhenAll(publishCCTasks);
}
private async Task <List <string> > UploadAsync(IAsyncCommandContext context, int uploaderId, CancellationToken token)
{
List <string> failedFiles = new List <string>();
string fileToUpload;
Stopwatch uploadTimer = new Stopwatch();
while (_fileUploadQueue.TryDequeue(out fileToUpload))
{
token.ThrowIfCancellationRequested();
try
{
using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read, FileShare.Read))
{
string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
uploadTimer.Restart();
bool catchExceptionDuringUpload = false;
HttpResponseMessage response = null;
try
{
response = await _fileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, cancellationToken : token, chunkSize : 4 * 1024 * 1024);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Output(StringUtil.Loc("FileUploadCancelled", fileToUpload));
if (response != null)
{
response.Dispose();
response = null;
}
throw;
}
catch (Exception ex)
{
catchExceptionDuringUpload = true;
context.Output(StringUtil.Loc("FileUploadFailed", fileToUpload, ex.Message));
context.Output(ex.ToString());
}
uploadTimer.Stop();
if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created))
{
if (response != null)
{
context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, fileToUpload, itemPath));
}
// output detail upload trace for the file.
ConcurrentQueue <string> logQueue;
if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue))
{
context.Output(StringUtil.Loc("FileUploadDetailTrace", itemPath));
string message;
while (logQueue.TryDequeue(out message))
{
context.Output(message);
}
}
// tracking file that failed to upload.
failedFiles.Add(fileToUpload);
}
else
{
context.Debug(StringUtil.Loc("FileUploadFinish", fileToUpload, uploadTimer.ElapsedMilliseconds));
// debug detail upload trace for the file.
ConcurrentQueue <string> logQueue;
if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue))
{
context.Debug($"Detail upload trace for file: {itemPath}");
string message;
while (logQueue.TryDequeue(out message))
{
context.Debug(message);
}
}
}
if (response != null)
{
response.Dispose();
response = null;
}
}
Interlocked.Increment(ref _filesProcessed);
}
catch (Exception ex)
{
context.Output(StringUtil.Loc("FileUploadFileOpenFailed", ex.Message, fileToUpload));
throw ex;
}
}
return(failedFiles);
}
private async Task ReportingAsync(IAsyncCommandContext context, int totalFiles, CancellationToken token)
{
while (_filesUploaded != totalFiles)
{
context.Output(StringUtil.Loc("FileUploadProgress", totalFiles, _filesUploaded));
await Task.Delay(2000, token);
}
}
private async Task <UploadResult> BlobUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token)
{
// return files that fail to upload and total artifact size
var uploadResult = new UploadResult();
// nothing needs to upload
if (files.Count == 0)
{
return(uploadResult);
}
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
var(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, token);
// Upload to blobstore
var results = await BlobStoreUtils.UploadBatchToBlobstore(verbose, files, (level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(BlobUploadAsync), context), (str) => context.Output(str), dedupClient, clientTelemetry, token, enableReporting : true);
// Associate with TFS
context.Output(StringUtil.Loc("AssociateFiles"));
var queue = new ConcurrentQueue <BlobFileInfo>();
foreach (var file in results.fileDedupIds)
{
queue.Enqueue(file);
}
// Start associate monitor
var uploadFinished = new TaskCompletionSource <int>();
var associateMonitor = AssociateReportingAsync(context, files.Count(), uploadFinished, token);
// Start parallel associate tasks.
var parallelAssociateTasks = new List <Task <UploadResult> >();
for (int uploader = 0; uploader < concurrentUploads; uploader++)
{
parallelAssociateTasks.Add(AssociateAsync(context, queue, token));
}
// Wait for parallel associate tasks to finish.
await Task.WhenAll(parallelAssociateTasks);
foreach (var associateTask in parallelAssociateTasks)
{
// record all failed files.
uploadResult.AddUploadResult(await associateTask);
}
// Stop monitor task
uploadFinished.SetResult(0);
await associateMonitor;
// report telemetry
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.PlanId), out var planId))
{
planId = Guid.Empty;
}
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.JobId), out var jobId))
{
jobId = Guid.Empty;
}
await clientTelemetry.CommitTelemetryUpload(planId, jobId);
return(uploadResult);
}
private async Task UpdateBuildNumberAsync(
IAsyncCommandContext context,
Uri projectCollection,
VssCredentials credentials,
Guid projectId,
int buildId,
string buildNumber,
CancellationToken cancellationToken)
{
BuildServer buildServer = new BuildServer(projectCollection, credentials, projectId);
var build = await buildServer.UpdateBuildNumber(buildId, buildNumber, cancellationToken);
context.Output(StringUtil.Loc("UpdateBuildNumberForBuild", build.BuildNumber, build.Id));
}
private async Task ParallelUploadAsync(IAsyncCommandContext context, int uploaderId, CancellationToken token)
{
string fileToUpload;
Stopwatch uploadTimer = new Stopwatch();
while (_fileUploadQueue.TryDequeue(out fileToUpload))
{
token.ThrowIfCancellationRequested();
try
{
context.Output(StringUtil.Loc("StartFileUpload", fileToUpload, new FileInfo(fileToUpload).Length));
using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read))
{
string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
uploadTimer.Restart();
var response = await FileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, token);
uploadTimer.Stop();
if (response.StatusCode != System.Net.HttpStatusCode.Created)
{
throw new Exception(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, fileToUpload, itemPath));
}
else
{
context.Output(StringUtil.Loc("FileUploadFinish", fileToUpload, uploadTimer.ElapsedMilliseconds));
}
}
Interlocked.Increment(ref _filesUploaded);
}
catch (Exception ex) when (!(ex is OperationCanceledException))
{
_exceptionsDuringFileUpload.Add(ex);
_uploadCancellationTokenSource.Cancel();
return;
}
}
}
private async Task <UploadResult> AssociateAsync(IAsyncCommandContext context, ConcurrentQueue <BlobFileInfo> associateQueue, CancellationToken token)
{
var uploadResult = new UploadResult();
var retryHelper = new RetryHelper(context);
var uploadTimer = new Stopwatch();
while (associateQueue.TryDequeue(out var file))
{
uploadTimer.Restart();
string itemPath = (_containerPath.TrimEnd('/') + "/" + file.Path.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
bool catchExceptionDuringUpload = false;
HttpResponseMessage response = null;
try
{
if (file.Success)
{
var length = (long)file.Node.TransitiveContentBytes;
response = await retryHelper.Retry(async() => await _fileContainerHttpClient.CreateItemForArtifactUpload(_containerId, itemPath, _projectId,
file.DedupId.ValueString, length, token),
(retryCounter) => (int)Math.Pow(retryCounter, 2) * 5,
(exception) => true);
uploadResult.TotalFileSizeUploaded += length;
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Output(StringUtil.Loc("FileUploadCancelled", itemPath));
if (response != null)
{
response.Dispose();
}
throw;
}
catch (Exception ex)
{
catchExceptionDuringUpload = true;
context.Output(StringUtil.Loc("FileUploadFailed", itemPath, ex.Message));
context.Output(ex.ToString());
}
if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created) || !file.Success)
{
if (response != null)
{
context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, file.Path, itemPath));
}
if (!file.Success)
{
context.Output(StringUtil.Loc("FileContainerUploadFailedBlob", file.Path, itemPath));
}
// tracking file that failed to upload.
uploadResult.FailedFiles.Add(file.Path);
}
else
{
context.Debug(StringUtil.Loc("FileUploadFinish", file.Path, uploadTimer.ElapsedMilliseconds));
}
if (response != null)
{
response.Dispose();
}
Interlocked.Increment(ref _filesProcessed);
}
return(uploadResult);
}
private async Task PublishCodeCoverageAsync(IExecutionContext executionContext, IAsyncCommandContext commandContext, ICodeCoveragePublisher codeCoveragePublisher, IEnumerable<CodeCoverageStatistics> coverageData,
string project, Guid projectId, long containerId, CancellationToken cancellationToken)
{
//step 2: publish code coverage summary to TFS
if (coverageData != null && coverageData.Count() > 0)
{
commandContext.Output(StringUtil.Loc("PublishingCodeCoverage"));
foreach (var coverage in coverageData)
{
commandContext.Output(StringUtil.Format(" {0}- {1} of {2} covered.", coverage.Label, coverage.Covered, coverage.Total));
}
await codeCoveragePublisher.PublishCodeCoverageSummaryAsync(coverageData, project, cancellationToken);
}
// step 3: publish code coverage files as build artifacts
string additionalCodeCoverageFilePath = null;
string destinationSummaryFile = null;
var newReportDirectory = _reportDirectory;
try
{
var filesToPublish = new List<Tuple<string, string>>();
if (!Directory.Exists(newReportDirectory))
{
if (!string.IsNullOrWhiteSpace(newReportDirectory))
{
// user gave a invalid report directory. Write warning and continue.
executionContext.Warning(StringUtil.Loc("DirectoryNotFound", newReportDirectory));
}
newReportDirectory = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.ReportDirectory);
Directory.CreateDirectory(newReportDirectory);
}
var summaryFileName = Path.GetFileName(_summaryFileLocation);
destinationSummaryFile = Path.Combine(newReportDirectory, CodeCoverageConstants.SummaryFileDirectory + _buildId, summaryFileName);
Directory.CreateDirectory(Path.GetDirectoryName(destinationSummaryFile));
File.Copy(_summaryFileLocation, destinationSummaryFile, true);
filesToPublish.Add(new Tuple<string, string>(newReportDirectory, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.ReportDirectory)));
if (_additionalCodeCoverageFiles != null && _additionalCodeCoverageFiles.Count != 0)
{
additionalCodeCoverageFilePath = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory);
CodeCoverageUtilities.CopyFilesFromFileListWithDirStructure(_additionalCodeCoverageFiles, ref additionalCodeCoverageFilePath);
filesToPublish.Add(new Tuple<string, string>(additionalCodeCoverageFilePath, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory)));
}
commandContext.Output(StringUtil.Loc("PublishingCodeCoverageFiles"));
await codeCoveragePublisher.PublishCodeCoverageFilesAsync(commandContext, projectId, containerId, filesToPublish, File.Exists(Path.Combine(newReportDirectory, CodeCoverageConstants.DefaultIndexFile)), cancellationToken);
}
catch (IOException ex)
{
executionContext.Warning(StringUtil.Loc("ErrorOccuredWhilePublishingCCFiles", ex.Message));
}
finally
{
// clean temporary files.
if (!string.IsNullOrEmpty(additionalCodeCoverageFilePath))
{
if (Directory.Exists(additionalCodeCoverageFilePath))
{
Directory.Delete(path: additionalCodeCoverageFilePath, recursive: true);
}
}
if (!string.IsNullOrEmpty(destinationSummaryFile))
{
var summaryFileDirectory = Path.GetDirectoryName(destinationSummaryFile);
if (Directory.Exists(summaryFileDirectory))
{
Directory.Delete(path: summaryFileDirectory, recursive: true);
}
}
if (!Directory.Exists(_reportDirectory))
{
if (Directory.Exists(newReportDirectory))
{
//delete the generated report directory
Directory.Delete(path: newReportDirectory, recursive: true);
}
}
}
}
private async Task ProcessPluginCommandAsync(IAsyncCommandContext context, AgentCommandPluginExecutionContext pluginContext, string plugin, Command command, CancellationToken token)
{
// Resolve the working directory.
string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
ArgUtil.Directory(workingDirectory, nameof(workingDirectory));
// Agent.PluginHost
string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
// Agent.PluginHost's arguments
string arguments = $"command \"{plugin}\"";
// Execute the process. Exit code 0 should always be returned.
// A non-zero exit code indicates infrastructural failure.
// Any content coming from STDERR will indicate the command process failed.
// We can't use ## command for plugin to communicate, since we are already processing ## command
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
object stderrLock = new object();
List <string> stderr = new List <string>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
context.Output(e.Data);
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
lock (stderrLock)
{
stderr.Add(e.Data);
};
};
var redirectStandardIn = new InputQueue <string>();
redirectStandardIn.Enqueue(JsonUtility.ToString(pluginContext));
int returnCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : file,
arguments : arguments,
environment : null,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
redirectStandardIn : redirectStandardIn,
cancellationToken : token);
if (returnCode != 0)
{
context.Output(string.Join(Environment.NewLine, stderr));
throw new ProcessExitCodeException(returnCode, file, arguments);
}
else if (stderr.Count > 0)
{
throw new InvalidOperationException(string.Join(Environment.NewLine, stderr));
}
else
{
// Everything works fine.
// Return code is 0.
// No STDERR comes out.
}
}
}
private async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(IAsyncCommandContext context, string itemPath, CancellationToken cancellationToken)
{
// Create chunks and identifier
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
// Setup upload session to keep file for at mimimum one day
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, (str) => context.Output(str));
var keepUntulRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = _dedupClient.CreateUploadSession(keepUntulRef, tracer, FileSystem.Instance);
// Upload the chunks
var uploadRecord = _blobTelemetry.CreateRecord <BuildArtifactActionRecord>((level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
await _blobTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
return(dedupId, rootNode.TransitiveContentBytes);
}
public RetryHelper(IAsyncCommandContext commandContext, int maxRetries = 3)
{
Debug = (str) => commandContext.Debug(str);
Warning = (str) => commandContext.Output(str);
MaxRetries = maxRetries;
}
private async Task <UploadResult> BlobUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token)
{
// return files that fail to upload and total artifact size
var uploadResult = new UploadResult();
// nothing needs to upload
if (files.Count == 0)
{
return(uploadResult);
}
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
try
{
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
int maxParallelism = context.GetHostContext().GetService <IConfigurationStore>().GetSettings().MaxDedupParallelism;
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, maxParallelism, token);
// Upload to blobstore
var results = await BlobStoreUtils.UploadBatchToBlobstore(verbose, files, (level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(BlobUploadAsync), context), (str) => context.Output(str), dedupClient, clientTelemetry, token, enableReporting : true);
// Associate with TFS
context.Output(StringUtil.Loc("AssociateFiles"));
var queue = new ConcurrentQueue <BlobFileInfo>();
foreach (var file in results.fileDedupIds)
{
queue.Enqueue(file);
}
// Start associate monitor
var uploadFinished = new TaskCompletionSource <int>();
var associateMonitor = AssociateReportingAsync(context, files.Count(), uploadFinished, token);
// Start parallel associate tasks.
var parallelAssociateTasks = new List <Task <UploadResult> >();
for (int uploader = 0; uploader < concurrentUploads; uploader++)
{
parallelAssociateTasks.Add(AssociateAsync(context, queue, token));
}
// Wait for parallel associate tasks to finish.
await Task.WhenAll(parallelAssociateTasks);
foreach (var associateTask in parallelAssociateTasks)
{
// record all failed files.
uploadResult.AddUploadResult(await associateTask);
}
// Stop monitor task
uploadFinished.SetResult(0);
await associateMonitor;
// report telemetry
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.PlanId), out var planId))
{
planId = Guid.Empty;
}
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.JobId), out var jobId))
{
jobId = Guid.Empty;
}
await clientTelemetry.CommitTelemetryUpload(planId, jobId);
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, this._connection.Uri.ToString(), context.Warn);
throw;
}
catch
{
var blobStoreHost = dedupClient.Client.BaseAddress.Host;
var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform();
var warningMessage = StringUtil.Loc("BlobStoreUploadWarning", blobStoreHost, allowListLink);
context.Warn(warningMessage);
throw;
}
return(uploadResult);
}
private async Task UploadArtifactAsync(
IAsyncCommandContext context,
Uri projectCollection,
VssCredentials credentials,
Guid projectId,
long containerId,
string containerPath,
int buildId,
string name,
Dictionary<string, string> propertiesDictionary,
string source,
CancellationToken cancellationToken)
{
FileContainerServer fileContainerHelper = new FileContainerServer(projectCollection, credentials, projectId, containerId, containerPath);
await fileContainerHelper.CopyToContainerAsync(context, source, cancellationToken);
string fileContainerFullPath = StringUtil.Format($"#/{containerId}/{containerPath}");
context.Output(StringUtil.Loc("UploadToFileContainer", source, fileContainerFullPath));
BuildServer buildHelper = new BuildServer(projectCollection, credentials, projectId);
var artifact = await buildHelper.AssociateArtifact(buildId, name, WellKnownArtifactResourceTypes.Container, fileContainerFullPath, propertiesDictionary, cancellationToken);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId));
}
private async Task AssociateArtifactAsync(
IAsyncCommandContext context,
Uri projectCollection,
VssCredentials credentials,
Guid projectId,
int buildId,
string name,
string type,
string data,
Dictionary<string, string> propertiesDictionary,
CancellationToken cancellationToken)
{
BuildServer buildHelper = new BuildServer(projectCollection, credentials, projectId);
var artifact = await buildHelper.AssociateArtifact(buildId, name, type, data, propertiesDictionary, cancellationToken);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId));
}
public async Task CopyToContainerAsync(
IAsyncCommandContext context,
String source,
CancellationToken cancellationToken)
{
//set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit
int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2);
//context.Output($"Max Concurrent Uploads {maxConcurrentUploads}");
List <String> files;
if (File.Exists(source))
{
files = new List <String>()
{
source
};
_sourceParentDirectory = Path.GetDirectoryName(source);
}
else
{
files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList();
_sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
context.Output(StringUtil.Loc("TotalUploadFiles", files.Count()));
using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// hook up reporting event from file container client.
_fileContainerHttpClient.UploadFileReportTrace += UploadFileTraceReportReceived;
_fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived;
try
{
// try upload all files for the first time.
List <string> failedFiles = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
if (failedFiles.Count == 0)
{
// all files have been upload succeed.
context.Output(StringUtil.Loc("FileUploadSucceed"));
return;
}
else
{
context.Output(StringUtil.Loc("FileUploadFailedRetryLater", failedFiles.Count));
}
// Delay 1 min then retry failed files.
for (int timer = 60; timer > 0; timer -= 5)
{
context.Output(StringUtil.Loc("FileUploadRetryInSecond", timer));
await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
}
// Retry upload all failed files.
context.Output(StringUtil.Loc("FileUploadRetry", failedFiles.Count));
failedFiles = await ParallelUploadAsync(context, failedFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
if (failedFiles.Count == 0)
{
// all files have been upload succeed after retry.
context.Output(StringUtil.Loc("FileUploadRetrySucceed"));
return;
}
else
{
throw new Exception(StringUtil.Loc("FileUploadFailedAfterRetry"));
}
}
finally
{
_fileContainerHttpClient.UploadFileReportTrace -= UploadFileTraceReportReceived;
_fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived;
}
}
}
private async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(IAsyncCommandContext context, string itemPath, CancellationToken cancellationToken)
{
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
return(await BlobStoreUtils.UploadToBlobStore(verbose, itemPath, (level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(UploadToBlobStore), context), (str) => context.Output(str), _dedupClient, _blobTelemetry, cancellationToken));
}
private async Task PublishCodeCoverageAsync(IExecutionContext executionContext, IAsyncCommandContext commandContext, ICodeCoveragePublisher codeCoveragePublisher, IEnumerable <CodeCoverageStatistics> coverageData,
string project, Guid projectId, long containerId, CancellationToken cancellationToken)
{
//step 2: publish code coverage summary to TFS
if (coverageData != null && coverageData.Count() > 0)
{
commandContext.Output(StringUtil.Loc("PublishingCodeCoverage"));
foreach (var coverage in coverageData)
{
commandContext.Output(StringUtil.Format(" {0}- {1} of {2} covered.", coverage.Label, coverage.Covered, coverage.Total));
}
await codeCoveragePublisher.PublishCodeCoverageSummaryAsync(coverageData, project, cancellationToken);
}
// step 3: publish code coverage files as build artifacts
string additionalCodeCoverageFilePath = null;
string destinationSummaryFile = null;
var newReportDirectory = _reportDirectory;
try
{
var filesToPublish = new List <Tuple <string, string> >();
if (!Directory.Exists(newReportDirectory))
{
if (!string.IsNullOrWhiteSpace(newReportDirectory))
{
// user gave a invalid report directory. Write warning and continue.
executionContext.Warning(StringUtil.Loc("DirectoryNotFound", newReportDirectory));
}
newReportDirectory = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.ReportDirectory);
Directory.CreateDirectory(newReportDirectory);
}
var summaryFileName = Path.GetFileName(_summaryFileLocation);
destinationSummaryFile = Path.Combine(newReportDirectory, CodeCoverageConstants.SummaryFileDirectory + _buildId, summaryFileName);
Directory.CreateDirectory(Path.GetDirectoryName(destinationSummaryFile));
File.Copy(_summaryFileLocation, destinationSummaryFile, true);
filesToPublish.Add(new Tuple <string, string>(newReportDirectory, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.ReportDirectory)));
if (_additionalCodeCoverageFiles != null && _additionalCodeCoverageFiles.Count != 0)
{
additionalCodeCoverageFilePath = GetCoverageDirectory(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory);
CodeCoverageUtilities.CopyFilesFromFileListWithDirStructure(_additionalCodeCoverageFiles, ref additionalCodeCoverageFilePath);
filesToPublish.Add(new Tuple <string, string>(additionalCodeCoverageFilePath, GetCoverageDirectoryName(_buildId.ToString(), CodeCoverageConstants.RawFilesDirectory)));
}
commandContext.Output(StringUtil.Loc("PublishingCodeCoverageFiles"));
await codeCoveragePublisher.PublishCodeCoverageFilesAsync(commandContext, projectId, containerId, filesToPublish, File.Exists(Path.Combine(newReportDirectory, CodeCoverageConstants.DefaultIndexFile)), cancellationToken);
}
catch (IOException ex)
{
executionContext.Warning(StringUtil.Loc("ErrorOccuredWhilePublishingCCFiles", ex.Message));
}
finally
{
// clean temporary files.
if (!string.IsNullOrEmpty(additionalCodeCoverageFilePath))
{
if (Directory.Exists(additionalCodeCoverageFilePath))
{
Directory.Delete(path: additionalCodeCoverageFilePath, recursive: true);
}
}
if (!string.IsNullOrEmpty(destinationSummaryFile))
{
var summaryFileDirectory = Path.GetDirectoryName(destinationSummaryFile);
if (Directory.Exists(summaryFileDirectory))
{
Directory.Delete(path: summaryFileDirectory, recursive: true);
}
}
if (!Directory.Exists(_reportDirectory))
{
if (Directory.Exists(newReportDirectory))
{
//delete the generated report directory
Directory.Delete(path: newReportDirectory, recursive: true);
}
}
}
}
private async Task AddBuildTagAsync(
IAsyncCommandContext context,
Uri projectCollection,
VssCredentials credentials,
Guid projectId,
int buildId,
string buildTag,
CancellationToken cancellationToken)
{
BuildServer buildServer = new BuildServer(projectCollection, credentials, projectId);
var tags = await buildServer.AddBuildTag(buildId, buildTag, cancellationToken);
if (tags == null || !tags.Contains(buildTag))
{
throw new Exception(StringUtil.Loc("BuildTagAddFailed", buildTag));
}
else
{
context.Output(StringUtil.Loc("BuildTagsForBuild", buildId, String.Join(", ", tags)));
}
}
private async Task <UploadResult> ParallelUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token)
{
// return files that fail to upload and total artifact size
var uploadResult = new UploadResult();
// nothing needs to upload
if (files.Count == 0)
{
return(uploadResult);
}
var uploadToBlob = String.Equals(context.GetVariableValueOrDefault("agent.UploadBuildArtifactsToBlob"), "true", StringComparison.InvariantCultureIgnoreCase);
if (uploadToBlob)
{
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
var(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, token);
_dedupClient = dedupClient;
_blobTelemetry = clientTelemetry;
}
// ensure the file upload queue is empty.
if (!_fileUploadQueue.IsEmpty)
{
throw new ArgumentOutOfRangeException(nameof(_fileUploadQueue));
}
// enqueue file into upload queue.
foreach (var file in files)
{
_fileUploadQueue.Enqueue(file);
}
// Start upload monitor task.
_filesProcessed = 0;
_uploadFinished = new TaskCompletionSource <int>();
_fileUploadTraceLog.Clear();
_fileUploadProgressLog.Clear();
Task uploadMonitor = ReportingAsync(context, files.Count(), _uploadCancellationTokenSource.Token);
// Start parallel upload tasks.
List <Task <UploadResult> > parallelUploadingTasks = new List <Task <UploadResult> >();
for (int uploader = 0; uploader < concurrentUploads; uploader++)
{
parallelUploadingTasks.Add(UploadAsync(context, uploader, uploadToBlob, _uploadCancellationTokenSource.Token));
}
// Wait for parallel upload finish.
await Task.WhenAll(parallelUploadingTasks);
foreach (var uploadTask in parallelUploadingTasks)
{
// record all failed files.
uploadResult.AddUploadResult(await uploadTask);
}
// Stop monitor task;
_uploadFinished.TrySetResult(0);
await uploadMonitor;
// report telemetry
if (uploadToBlob)
{
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.PlanId), out var planId))
{
planId = Guid.Empty;
}
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.JobId), out var jobId))
{
jobId = Guid.Empty;
}
await _blobTelemetry.CommitTelemetry(planId, jobId);
}
return(uploadResult);
}
public async Task <long> CopyToContainerAsync(
IAsyncCommandContext context,
String source,
CancellationToken cancellationToken)
{
ArgUtil.NotNull(context, nameof(context));
ArgUtil.NotNull(source, nameof(source));
//set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit
int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2);
//context.Output($"Max Concurrent Uploads {maxConcurrentUploads}");
List <String> files;
if (File.Exists(source))
{
files = new List <String>()
{
source
};
_sourceParentDirectory = Path.GetDirectoryName(source);
}
else
{
files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList();
_sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
context.Output(StringUtil.Loc("TotalUploadFiles", files.Count()));
using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// hook up reporting event from file container client.
_fileContainerHttpClient.UploadFileReportTrace += UploadFileTraceReportReceived;
_fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived;
try
{
var uploadToBlob = String.Equals(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.UploadBuildArtifactsToBlob), "true", StringComparison.InvariantCultureIgnoreCase) &&
!AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
// try upload all files for the first time.
UploadResult uploadResult = null;
if (uploadToBlob)
{
try
{
uploadResult = await BlobUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
}
catch
{
// Fall back to FCS upload if we cannot upload to blob
context.Warn(StringUtil.Loc("BlobStoreUploadWarning"));
uploadToBlob = false;
}
}
if (!uploadToBlob)
{
uploadResult = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
}
if (uploadResult.FailedFiles.Count == 0)
{
// all files have been upload succeed.
context.Output(StringUtil.Loc("FileUploadSucceed"));
return(uploadResult.TotalFileSizeUploaded);
}
else
{
context.Output(StringUtil.Loc("FileUploadFailedRetryLater", uploadResult.FailedFiles.Count));
}
// Delay 1 min then retry failed files.
for (int timer = 60; timer > 0; timer -= 5)
{
context.Output(StringUtil.Loc("FileUploadRetryInSecond", timer));
await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
}
// Retry upload all failed files.
context.Output(StringUtil.Loc("FileUploadRetry", uploadResult.FailedFiles.Count));
UploadResult retryUploadResult;
if (uploadToBlob)
{
retryUploadResult = await BlobUploadAsync(context, uploadResult.FailedFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
}
else
{
retryUploadResult = await ParallelUploadAsync(context, uploadResult.FailedFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
}
if (retryUploadResult.FailedFiles.Count == 0)
{
// all files have been upload succeed after retry.
context.Output(StringUtil.Loc("FileUploadRetrySucceed"));
return(uploadResult.TotalFileSizeUploaded + retryUploadResult.TotalFileSizeUploaded);
}
else
{
throw new Exception(StringUtil.Loc("FileUploadFailedAfterRetry"));
}
}
finally
{
_fileContainerHttpClient.UploadFileReportTrace -= UploadFileTraceReportReceived;
_fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived;
}
}
}