public async Task <long> CopyToContainerAsync(
IAsyncCommandContext context,
String source,
CancellationToken cancellationToken)
{
ArgUtil.NotNull(context, nameof(context));
ArgUtil.NotNull(source, nameof(source));
//set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit
int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2);
//context.Output($"Max Concurrent Uploads {maxConcurrentUploads}");
List <String> files;
if (File.Exists(source))
{
files = new List <String>()
{
source
};
_sourceParentDirectory = Path.GetDirectoryName(source);
}
else
{
files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList();
_sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
}
context.Output(StringUtil.Loc("TotalUploadFiles", files.Count()));
using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// hook up reporting event from file container client.
_fileContainerHttpClient.UploadFileReportTrace += UploadFileTraceReportReceived;
_fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived;
try
{
var uploadToBlob = String.Equals(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.UploadBuildArtifactsToBlob), "true", StringComparison.InvariantCultureIgnoreCase) &&
!AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
// try upload all files for the first time.
UploadResult uploadResult = null;
if (uploadToBlob)
{
try
{
uploadResult = await BlobUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
}
catch
{
// Fall back to FCS upload if we cannot upload to blob
context.Warn(StringUtil.Loc("BlobStoreUploadWarning"));
uploadToBlob = false;
}
}
if (!uploadToBlob)
{
uploadResult = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
}
if (uploadResult.FailedFiles.Count == 0)
{
// all files have been upload succeed.
context.Output(StringUtil.Loc("FileUploadSucceed"));
return(uploadResult.TotalFileSizeUploaded);
}
else
{
context.Output(StringUtil.Loc("FileUploadFailedRetryLater", uploadResult.FailedFiles.Count));
}
// Delay 1 min then retry failed files.
for (int timer = 60; timer > 0; timer -= 5)
{
context.Output(StringUtil.Loc("FileUploadRetryInSecond", timer));
await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
}
// Retry upload all failed files.
context.Output(StringUtil.Loc("FileUploadRetry", uploadResult.FailedFiles.Count));
UploadResult retryUploadResult;
if (uploadToBlob)
{
retryUploadResult = await BlobUploadAsync(context, uploadResult.FailedFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
}
else
{
retryUploadResult = await ParallelUploadAsync(context, uploadResult.FailedFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token);
}
if (retryUploadResult.FailedFiles.Count == 0)
{
// all files have been upload succeed after retry.
context.Output(StringUtil.Loc("FileUploadRetrySucceed"));
return(uploadResult.TotalFileSizeUploaded + retryUploadResult.TotalFileSizeUploaded);
}
else
{
throw new Exception(StringUtil.Loc("FileUploadFailedAfterRetry"));
}
}
finally
{
_fileContainerHttpClient.UploadFileReportTrace -= UploadFileTraceReportReceived;
_fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived;
}
}
}
private async Task <UploadResult> BlobUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token)
{
// return files that fail to upload and total artifact size
var uploadResult = new UploadResult();
// nothing needs to upload
if (files.Count == 0)
{
return(uploadResult);
}
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
try
{
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
int maxParallelism = context.GetHostContext().GetService <IConfigurationStore>().GetSettings().MaxDedupParallelism;
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, maxParallelism, token);
// Upload to blobstore
var results = await BlobStoreUtils.UploadBatchToBlobstore(verbose, files, (level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(BlobUploadAsync), context), (str) => context.Output(str), dedupClient, clientTelemetry, token, enableReporting : true);
// Associate with TFS
context.Output(StringUtil.Loc("AssociateFiles"));
var queue = new ConcurrentQueue <BlobFileInfo>();
foreach (var file in results.fileDedupIds)
{
queue.Enqueue(file);
}
// Start associate monitor
var uploadFinished = new TaskCompletionSource <int>();
var associateMonitor = AssociateReportingAsync(context, files.Count(), uploadFinished, token);
// Start parallel associate tasks.
var parallelAssociateTasks = new List <Task <UploadResult> >();
for (int uploader = 0; uploader < concurrentUploads; uploader++)
{
parallelAssociateTasks.Add(AssociateAsync(context, queue, token));
}
// Wait for parallel associate tasks to finish.
await Task.WhenAll(parallelAssociateTasks);
foreach (var associateTask in parallelAssociateTasks)
{
// record all failed files.
uploadResult.AddUploadResult(await associateTask);
}
// Stop monitor task
uploadFinished.SetResult(0);
await associateMonitor;
// report telemetry
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.PlanId), out var planId))
{
planId = Guid.Empty;
}
if (!Guid.TryParse(context.GetVariableValueOrDefault(WellKnownDistributedTaskVariables.JobId), out var jobId))
{
jobId = Guid.Empty;
}
await clientTelemetry.CommitTelemetryUpload(planId, jobId);
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, this._connection.Uri.ToString(), context.Warn);
throw;
}
catch
{
var blobStoreHost = dedupClient.Client.BaseAddress.Host;
var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform();
var warningMessage = StringUtil.Loc("BlobStoreUploadWarning", blobStoreHost, allowListLink);
context.Warn(warningMessage);
throw;
}
return(uploadResult);
}