internal static async Task <DedupNode> GetDedupNodeFromFileAsync(string path, IFileSystem fileSystem, CancellationToken token)
{
var dedupNode = await ChunkerHelper.CreateFromFileAsync(
fileSystem : fileSystem,
path : path,
cancellationToken : token,
configureAwait : false);
return(dedupNode);
}
private async Task <DedupNode> GetDedupNodeFromFileAsync(string path, CancellationToken cts)
{
var dedupNode = await ChunkerHelper.CreateFromFileAsync(
fileSystem : _artifactFileSystem,
path : path,
cancellationToken : cts,
configureAwait : false);
return(dedupNode);
}
public async Task <(DedupIdentifier dedupId, ulong length)> UploadAttachmentToBlobStore(bool verbose, string itemPath, Guid planId, Guid jobId, CancellationToken cancellationToken)
{
UploadedAttachmentBlobFiles.Add(itemPath);
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
return(dedupId, rootNode.TransitiveContentBytes);
}
public static async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(
bool verbose,
string itemPath,
Func <TelemetryInformationLevel, Uri, string, BlobStoreTelemetryRecord> telemetryRecordFactory,
Action <string> traceOutput,
DedupStoreClient dedupClient,
BlobStoreClientTelemetry clientTelemetry,
CancellationToken cancellationToken)
{
// Create chunks and identifier
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
// ChunkHelper uses 64k block default size
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
// Setup upload session to keep file for at mimimum one day
// Blobs will need to be associated with the server with an ID ref otherwise they will be
// garbage collected after one day
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, traceOutput);
var keepUntilRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = dedupClient.CreateUploadSession(keepUntilRef, tracer, FileSystem.Instance);
// Upload the chunks
var uploadRecord = clientTelemetry.CreateRecord <BlobStoreTelemetryRecord>(telemetryRecordFactory);
await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken);
return(uploadSession.UploadStatistics);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
return(dedupId, rootNode.TransitiveContentBytes);
}
private async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(IAsyncCommandContext context, string itemPath, CancellationToken cancellationToken)
{
// Create chunks and identifier
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
// Setup upload session to keep file for at mimimum one day
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, (str) => context.Output(str));
var keepUntulRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = _dedupClient.CreateUploadSession(keepUntulRef, tracer, FileSystem.Instance);
// Upload the chunks
var uploadRecord = _blobTelemetry.CreateRecord <BuildArtifactActionRecord>((level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
await _blobTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
return(dedupId, rootNode.TransitiveContentBytes);
}
private static async Task <List <BlobFileInfo> > GenerateHashes(IReadOnlyList <string> filePaths, CancellationToken cancellationToken)
{
var nodes = new BlobFileInfo[filePaths.Count];
var queue = NonSwallowingActionBlock.Create <int>(
async i =>
{
var itemPath = filePaths[i];
try
{
var dedupNode = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
nodes[i] = new BlobFileInfo
{
Path = itemPath,
Node = dedupNode,
Success = dedupNode != null
};
}
catch (Exception)
{
nodes[i] = new BlobFileInfo
{
Path = itemPath,
Success = false
};
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = cancellationToken,
});
await queue.SendAllAndCompleteSingleBlockNetworkAsync(Enumerable.Range(0, filePaths.Count), cancellationToken);
return(nodes.ToList());
}
