internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
IEnumerable <string> key,
string path,
string salt,
string variableToSetOnHit,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var pipelineCacheClient = this.CreateClient(context, connection);
GetPipelineCacheArtifactOptions options = new GetPipelineCacheArtifactOptions
{
Key = key,
Scope = "myscope",
Salt = salt,
};
var result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(options, cancellationToken);
if (result == null)
{
return;
}
else
{
Console.WriteLine("Manifest ID is: {0}", result.ManifestId.ValueString);
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection);
await this.DownloadPipelineCacheAsync(dedupManifestClient, result.ManifestId, path, cancellationToken);
context.SetVariable($"{PipelineCacheVarPrefix}.{variableToSetOnHit}", "True");
Console.WriteLine("Cache restored.");
}
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
IEnumerable <string> key,
string path,
string salt,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection);
var result = await dedupManifestClient.PublishAsync(path, cancellationToken);
var scope = "myscope";
CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions
{
Key = key,
RootId = result.RootId,
ManifestId = result.ManifestId,
Scope = scope,
ProofNodes = result.ProofNodes.ToArray(),
Salt = salt
};
var pipelineCacheClient = this.CreateClient(context, connection);
await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken);
Console.WriteLine("Saved item.");
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint fingerprint,
string path,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
// Check if the key exists.
PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
//If cache exists, return.
if (getResult != null)
{
context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
return;
}
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(path, cancellationToken));
});
CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions
{
Fingerprint = fingerprint,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
};
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
IEnumerable <string> key,
string path,
string salt,
string variableToSetOnHit,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
GetPipelineCacheArtifactOptions options = new GetPipelineCacheArtifactOptions
{
Key = key,
Salt = salt,
};
using (clientTelemetry)
{
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
if (result == null)
{
return;
}
else
{
context.Output($"Manifest ID is: {result.ManifestId.ValueString}");
PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await this.DownloadPipelineCacheAsync(dedupManifestClient, result.ManifestId, path, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord);
if (!string.IsNullOrEmpty(variableToSetOnHit))
{
context.SetVariable($"{PipelineArtifactConstants.PipelineCache}.{variableToSetOnHit}", "True");
}
Console.WriteLine("Cache restored.");
}
}
}
public static async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(
bool verbose,
string itemPath,
Func <TelemetryInformationLevel, Uri, string, BlobStoreTelemetryRecord> telemetryRecordFactory,
Action <string> traceOutput,
DedupStoreClient dedupClient,
BlobStoreClientTelemetry clientTelemetry,
CancellationToken cancellationToken)
{
// Create chunks and identifier
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
// ChunkHelper uses 64k block default size
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
// Setup upload session to keep file for at mimimum one day
// Blobs will need to be associated with the server with an ID ref otherwise they will be
// garbage collected after one day
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, traceOutput);
var keepUntilRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = dedupClient.CreateUploadSession(keepUntilRef, tracer, FileSystem.Instance);
// Upload the chunks
var uploadRecord = clientTelemetry.CreateRecord <BlobStoreTelemetryRecord>(telemetryRecordFactory);
await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken);
return(uploadSession.UploadStatistics);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
return(dedupId, rootNode.TransitiveContentBytes);
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
IEnumerable <string> key,
string path,
string salt,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(path, cancellationToken));
});
CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions
{
Key = key,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
Salt = salt
};
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
private async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(IAsyncCommandContext context, string itemPath, CancellationToken cancellationToken)
{
// Create chunks and identifier
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
// Setup upload session to keep file for at mimimum one day
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, (str) => context.Output(str));
var keepUntulRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = _dedupClient.CreateUploadSession(keepUntulRef, tracer, FileSystem.Instance);
// Upload the chunks
var uploadRecord = _blobTelemetry.CreateRecord <BuildArtifactActionRecord>((level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
await _blobTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
return(dedupId, rootNode.TransitiveContentBytes);
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint fingerprint,
string path,
CancellationToken cancellationToken,
string preferredContentFormat = ContentFormatConstants.Files)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
// Check if the key exists.
PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
//If cache exists, return.
if (getResult != null)
{
context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
return;
}
string uploadPath = await this.GetUploadPathAsync(preferredContentFormat, context, path, cancellationToken);
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken));
});
CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract
{
Fingerprint = fingerprint,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
ContentFormat = preferredContentFormat
};
// delete archive file if it's tar.
if (string.Equals(preferredContentFormat, ContentFormatConstants.SingleTar, StringComparison.OrdinalIgnoreCase))
{
try
{
if (File.Exists(uploadPath))
{
File.Delete(uploadPath);
}
}
catch { }
}
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint[] fingerprints,
string path,
string cacheHitVariable,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
if (result != null)
{
context.Output($"Entry found at fingerprint: `{result.Fingerprint.ToString()}`");
context.Verbose($"Manifest ID is: {result.ManifestId.ValueString}");
PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, path, result.ContentFormat, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord);
context.Output("Cache restored.");
}
if (!string.IsNullOrEmpty(cacheHitVariable))
{
if (result == null)
{
context.SetVariable(cacheHitVariable, "false");
}
else
{
context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`");
bool foundExact = false;
foreach (var fingerprint in fingerprints)
{
context.Verbose($"This fingerprint: `{fingerprint.ToString()}`");
if (fingerprint == result.Fingerprint ||
result.Fingerprint.Segments.Length == 1 && result.Fingerprint.Segments.Single() == fingerprint.SummarizeForV1())
{
foundExact = true;
break;
}
}
context.SetVariable(cacheHitVariable, foundExact ? "true" : "inexact");
}
}
}
}
public static async Task <(List <BlobFileInfo> fileDedupIds, ulong length)> UploadBatchToBlobstore(
bool verbose,
IReadOnlyList <string> itemPaths,
Func <TelemetryInformationLevel, Uri, string, BlobStoreTelemetryRecord> telemetryRecordFactory,
Action <string> traceOutput,
DedupStoreClient dedupClient,
BlobStoreClientTelemetry clientTelemetry,
CancellationToken cancellationToken,
bool enableReporting = false)
{
// Create chunks and identifier
traceOutput(StringUtil.Loc("BuildingFileTree"));
var fileNodes = await GenerateHashes(itemPaths, cancellationToken);
var rootNode = CreateNodeToUpload(fileNodes.Where(x => x.Success).Select(y => y.Node));
// If there are multiple paths to one DedupId (duplicate files)
// take the last one
var fileDedupIds = new Dictionary <DedupIdentifier, string>();
foreach (var file in fileNodes.Where(x => x.Success))
{
// ChunkHelper uses 64k block default size
var dedupId = file.Node.GetDedupIdentifier(HashType.Dedup64K);
fileDedupIds[dedupId] = file.Path;
}
// Setup upload session to keep file for at mimimum one day
// Blobs will need to be associated with the server with an ID ref otherwise they will be
// garbage collected after one day
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, traceOutput);
var keepUntilRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = dedupClient.CreateUploadSession(keepUntilRef, tracer, FileSystem.Instance);
using (var reportingCancelSrc = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// Log stats
Task reportingTask = null;
if (enableReporting)
{
reportingTask = StartReportingTask(traceOutput, (long)rootNode.TransitiveContentBytes, uploadSession, reportingCancelSrc);
}
// Upload the chunks
var uploadRecord = clientTelemetry.CreateRecord <BlobStoreTelemetryRecord>(telemetryRecordFactory);
await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
await uploadSession.UploadAsync(rootNode, fileDedupIds, cancellationToken);
return(uploadSession.UploadStatistics);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
if (enableReporting)
{
reportingCancelSrc.Cancel();
await reportingTask;
}
}
return(fileNodes, rootNode.TransitiveContentBytes);
}
