public FileShareActionRecord(TelemetryInformationLevel level, Uri baseAddress, string eventNamePrefix, string eventNameSuffix, AgentTaskPluginExecutionContext context, uint attemptNumber = 1)
: base(level, baseAddress, eventNamePrefix, eventNameSuffix, context, attemptNumber)
{
}
public PipelineCacheServer(AgentTaskPluginExecutionContext context)
{
this.tracer = context.CreateArtifactsTracer();
}
protected override async Task ProcessCommandInternalAsync(
AgentTaskPluginExecutionContext context,
string targetPath,
CancellationToken token)
{
string artifactName = this.GetArtifactName(context);
// Create target directory if absent
string fullPath = Path.GetFullPath(targetPath);
bool isDir = Directory.Exists(fullPath);
if (!isDir)
{
Directory.CreateDirectory(fullPath);
}
// Project ID
// TODO: use a constant for project id, which is currently defined in Microsoft.VisualStudio.Services.Agent.Constants.Variables.System.TeamProjectId (Ting)
string guidStr = context.Variables.GetValueOrDefault("system.teamProjectId")?.Value;
Guid.TryParse(guidStr, out Guid projectId);
ArgUtil.NotEmpty(projectId, nameof(projectId));
// Build ID
int buildId = 0;
string buildIdStr = context.GetInput(ArtifactEventProperties.PipelineId, required: false);
// Determine the build id
if (Int32.TryParse(buildIdStr, out buildId) && buildId != 0)
{
// A) Build Id provided by user input
context.Output(StringUtil.Loc("DownloadingFromBuild", buildId));
}
else
{
// B) Build Id provided by environment
buildIdStr = context.Variables.GetValueOrDefault(BuildVariables.BuildId)?.Value ?? string.Empty;
if (int.TryParse(buildIdStr, out buildId) && buildId != 0)
{
context.Output(StringUtil.Loc("DownloadingFromBuild", buildId));
}
else
{
string hostType = context.Variables.GetValueOrDefault("system.hosttype")?.Value;
if (string.Equals(hostType, "Release", StringComparison.OrdinalIgnoreCase) ||
string.Equals(hostType, "DeploymentGroup", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("BuildIdIsNotAvailable", hostType ?? string.Empty, hostType ?? string.Empty));
}
else if (!string.Equals(hostType, "Build", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("CannotDownloadFromCurrentEnvironment", hostType ?? string.Empty));
}
else
{
// This should not happen since the build id comes from build environment. But a user may override that so we must be careful.
throw new ArgumentException(StringUtil.Loc("BuildIdIsNotValid", buildIdStr));
}
}
}
// Download from VSTS BlobStore
context.Output(StringUtil.Loc("DownloadArtifactTo", targetPath));
PipelineArtifactServer server = new PipelineArtifactServer(tracer);
await server.DownloadAsync(context, projectId, buildId, artifactName, targetPath, token);
context.Output(StringUtil.Loc("DownloadArtifactFinished"));
}
protected abstract Task ProcessCommandInternalAsync(
AgentTaskPluginExecutionContext context,
CancellationToken token);
protected override async Task ProcessCommandInternalAsync(
AgentTaskPluginExecutionContext context,
CancellationToken token)
{
ArgUtil.NotNull(context, nameof(context));
string artifactName = context.GetInput(ArtifactEventProperties.ArtifactName, required: false);
string branchName = context.GetInput(ArtifactEventProperties.BranchName, required: false);
string pipelineDefinition = context.GetInput(ArtifactEventProperties.PipelineDefinition, required: false);
string sourceRun = context.GetInput(ArtifactEventProperties.SourceRun, required: true);
string pipelineTriggering = context.GetInput(ArtifactEventProperties.PipelineTriggering, required: false);
string pipelineVersionToDownload = context.GetInput(ArtifactEventProperties.PipelineVersionToDownload, required: false);
string targetPath = context.GetInput(DownloadPath, required: true);
string environmentBuildId = context.Variables.GetValueOrDefault(BuildVariables.BuildId)?.Value ?? string.Empty; // BuildID provided by environment.
string itemPattern = context.GetInput(ArtifactEventProperties.ItemPattern, required: false);
string projectName = context.GetInput(ArtifactEventProperties.Project, required: false);
string tags = context.GetInput(ArtifactEventProperties.Tags, required: false);
string allowPartiallySucceededBuilds = context.GetInput(ArtifactEventProperties.AllowPartiallySucceededBuilds, required: false);
string allowFailedBuilds = context.GetInput(ArtifactEventProperties.AllowFailedBuilds, required: false);
string allowCanceledBuilds = context.GetInput(ArtifactEventProperties.AllowCanceledBuilds, required: false);
string userSpecifiedRunId = context.GetInput(RunId, required: false);
string defaultWorkingDirectory = context.Variables.GetValueOrDefault("system.defaultworkingdirectory").Value;
targetPath = Path.IsPathFullyQualified(targetPath) ? targetPath : Path.GetFullPath(Path.Combine(defaultWorkingDirectory, targetPath));
bool onPrem = !String.Equals(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.ServerType)?.Value, "Hosted", StringComparison.OrdinalIgnoreCase);
if (onPrem)
{
throw new InvalidOperationException(StringUtil.Loc("OnPremIsNotSupported"));
}
if (!PipelineArtifactPathHelper.IsValidArtifactName(artifactName))
{
throw new ArgumentException(StringUtil.Loc("ArtifactNameIsNotValid", artifactName));
}
string[] minimatchPatterns = itemPattern.Split(
new[] { "\n" },
StringSplitOptions.RemoveEmptyEntries
);
string[] tagsInput = tags.Split(
new[] { "," },
StringSplitOptions.None
);
if (!bool.TryParse(allowPartiallySucceededBuilds, out var allowPartiallySucceededBuildsBool))
{
allowPartiallySucceededBuildsBool = false;
}
if (!bool.TryParse(allowFailedBuilds, out var allowFailedBuildsBool))
{
allowFailedBuildsBool = false;
}
if (!bool.TryParse(allowCanceledBuilds, out var allowCanceledBuildsBool))
{
allowCanceledBuildsBool = false;
}
var resultFilter = GetResultFilter(allowPartiallySucceededBuildsBool, allowFailedBuildsBool, allowCanceledBuildsBool);
PipelineArtifactServer server = new PipelineArtifactServer(tracer);
PipelineArtifactDownloadParameters downloadParameters;
if (sourceRun == sourceRunCurrent)
{
// TODO: use a constant for project id, which is currently defined in Microsoft.VisualStudio.Services.Agent.Constants.Variables.System.TeamProjectId (Ting)
string projectIdStr = context.Variables.GetValueOrDefault("system.teamProjectId")?.Value;
if (String.IsNullOrEmpty(projectIdStr))
{
throw new ArgumentNullException(StringUtil.Loc("CannotBeNullOrEmpty"), "Project ID");
}
Guid projectId = Guid.Parse(projectIdStr);
ArgUtil.NotEmpty(projectId, nameof(projectId));
int pipelineId = 0;
if (int.TryParse(environmentBuildId, out pipelineId) && pipelineId != 0)
{
OutputBuildInfo(context, pipelineId);
}
else
{
string hostType = context.Variables.GetValueOrDefault("system.hosttype")?.Value;
if (string.Equals(hostType, "Release", StringComparison.OrdinalIgnoreCase) ||
string.Equals(hostType, "DeploymentGroup", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("BuildIdIsNotAvailable", hostType ?? string.Empty, hostType ?? string.Empty));
}
else if (!string.Equals(hostType, "Build", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("CannotDownloadFromCurrentEnvironment", hostType ?? string.Empty));
}
else
{
// This should not happen since the build id comes from build environment. But a user may override that so we must be careful.
throw new ArgumentException(StringUtil.Loc("BuildIdIsNotValid", environmentBuildId));
}
}
downloadParameters = new PipelineArtifactDownloadParameters
{
ProjectRetrievalOptions = BuildArtifactRetrievalOptions.RetrieveByProjectId,
ProjectId = projectId,
PipelineId = pipelineId,
ArtifactName = artifactName,
TargetDirectory = targetPath,
MinimatchFilters = minimatchPatterns,
MinimatchFilterWithArtifactName = true
};
}
else if (sourceRun == sourceRunSpecific)
{
if (String.IsNullOrEmpty(projectName))
{
throw new ArgumentNullException(StringUtil.Loc("CannotBeNullOrEmpty"), "Project Name");
}
Guid projectId;
bool isProjGuid = Guid.TryParse(projectName, out projectId);
if (!isProjGuid)
{
projectId = await GetProjectIdAsync(context, projectName);
}
// Set the default pipelineId to 0, which is an invalid build id and it has to be reassigned to a valid build id.
int pipelineId = 0;
bool pipelineTriggeringBool = false;
if (bool.TryParse(pipelineTriggering, out pipelineTriggeringBool) && pipelineTriggeringBool)
{
string triggeringPipeline = context.Variables.GetValueOrDefault("build.triggeredBy.buildId")?.Value;
if (!string.IsNullOrEmpty(triggeringPipeline))
{
pipelineId = int.Parse(triggeringPipeline);
}
}
if (pipelineId == 0)
{
if (pipelineVersionToDownload == pipelineVersionToDownloadLatest)
{
pipelineId = await this.GetPipelineIdAsync(context, pipelineDefinition, pipelineVersionToDownload, projectId.ToString(), tagsInput, resultFilter, null, cancellationToken : token);
}
else if (pipelineVersionToDownload == pipelineVersionToDownloadSpecific)
{
bool isPipelineIdNum = Int32.TryParse(userSpecifiedRunId, out pipelineId);
if (!isPipelineIdNum)
{
throw new ArgumentException(StringUtil.Loc("RunIDNotValid", userSpecifiedRunId));
}
}
else if (pipelineVersionToDownload == pipelineVersionToDownloadLatestFromBranch)
{
pipelineId = await this.GetPipelineIdAsync(context, pipelineDefinition, pipelineVersionToDownload, projectId.ToString(), tagsInput, resultFilter, branchName, cancellationToken : token);
}
else
{
throw new InvalidOperationException("Unreachable code!");
}
}
OutputBuildInfo(context, pipelineId);
downloadParameters = new PipelineArtifactDownloadParameters
{
ProjectRetrievalOptions = BuildArtifactRetrievalOptions.RetrieveByProjectName,
ProjectName = projectName,
ProjectId = projectId,
PipelineId = pipelineId,
ArtifactName = artifactName,
TargetDirectory = targetPath,
MinimatchFilters = minimatchPatterns,
MinimatchFilterWithArtifactName = true
};
}
else
{
throw new InvalidOperationException($"Build type '{sourceRun}' is not recognized.");
}
string fullPath = this.CreateDirectoryIfDoesntExist(targetPath);
DownloadOptions downloadOptions;
if (string.IsNullOrEmpty(downloadParameters.ArtifactName))
{
downloadOptions = DownloadOptions.MultiDownload;
}
else
{
downloadOptions = DownloadOptions.SingleDownload;
}
context.Output(StringUtil.Loc("DownloadArtifactTo", targetPath));
await server.DownloadAsyncV2(context, downloadParameters, downloadOptions, token);
context.Output(StringUtil.Loc("DownloadArtifactFinished"));
}
public Task PostJobCleanupAsync(AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository)
{
return(Task.CompletedTask);
}
// Process the command with preprocessed arguments.
protected abstract Task ProcessCommandInternalAsync(
AgentTaskPluginExecutionContext context,
string targetPath,
string artifactName,
CancellationToken token);
private static async Task RunProcessAsync(
AgentTaskPluginExecutionContext context,
string processFileName,
string processArguments,
Func <Process, CancellationToken, Task> additionalTaskToExecuteWhilstRunningProcess,
Action actionOnFailure,
CancellationToken cancellationToken)
{
var processTcs = new TaskCompletionSource <int>();
using (var cancelSource = new CancellationTokenSource())
using (var linkedSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, cancelSource.Token))
using (var process = new Process())
{
SetProcessStartInfo(process, processFileName, processArguments);
process.EnableRaisingEvents = true;
process.Exited += (sender, args) =>
{
cancelSource.Cancel();
processTcs.SetResult(process.ExitCode);
};
try
{
context.Debug($"Starting '{process.StartInfo.FileName}' with arguments '{process.StartInfo.Arguments}'...");
process.Start();
}
catch (Exception e)
{
ExceptionDispatchInfo.Capture(e).Throw();
}
var output = new List <string>();
Task readLines(string prefix, StreamReader reader) => Task.Run(async() =>
{
string line;
while (null != (line = await reader.ReadLineAsync()))
{
lock (output)
{
output.Add($"{prefix}{line}");
}
}
});
Task readStdOut = readLines("stdout: ", process.StandardOutput);
Task readStdError = readLines("stderr: ", process.StandardError);
// Our goal is to always have the process ended or killed by the time we exit the function.
try
{
using (cancellationToken.Register(() => process.Kill()))
{
// readStdOut and readStdError should only fail if the process dies
// processTcs.Task cannot fail as we only call SetResult on processTcs
IEnumerable <Task> tasks = new List <Task>
{
readStdOut,
readStdError,
processTcs.Task,
additionalTaskToExecuteWhilstRunningProcess(process, linkedSource.Token)
};
await Task.WhenAll(tasks);
}
int exitCode = await processTcs.Task;
if (exitCode == 0)
{
context.Output($"Process exit code: {exitCode}");
foreach (string line in output)
{
context.Output(line);
}
}
else
{
throw new Exception($"Process returned non-zero exit code: {exitCode}");
}
}
catch (Exception e)
{
actionOnFailure();
foreach (string line in output)
{
context.Error(line);
}
ExceptionDispatchInfo.Capture(e).Throw();
}
}
}
public Task <(DedupManifestArtifactClient client, BlobStoreClientTelemetry telemetry)> CreateDedupManifestClientAsync(AgentTaskPluginExecutionContext context, VssConnection connection, CancellationToken cancellationToken)
{
telemetrySender = new TestTelemetrySender();
return(Task.FromResult((client: (DedupManifestArtifactClient)null, telemetry: new BlobStoreClientTelemetry(
NoopAppTraceSource.Instance,
baseAddress,
telemetrySender))));
}
private CallbackAppTraceSource CreateTracer(AgentTaskPluginExecutionContext context)
{
var tracer = new CallbackAppTraceSource(str => context.Output(str), System.Diagnostics.SourceLevels.Information);
return(tracer);
}
// Download with minimatch patterns, V1.
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
PipelineArtifactDownloadParameters downloadParameters,
DownloadOptions downloadOptions,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
BuildServer buildHelper = new BuildServer(connection);
using (clientTelemetry)
{
// download all pipeline artifacts if artifact name is missing
if (downloadOptions == DownloadOptions.MultiDownload)
{
List <BuildArtifact> artifacts;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
artifacts = await buildHelper.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
IEnumerable <BuildArtifact> fileShareArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.FileShareArtifact, StringComparison.OrdinalIgnoreCase));
if (fileShareArtifacts.Any())
{
throw new InvalidOperationException("File Share is not supported in the Download Pipeline Artifact V1 task. Please use Download Pipeline Artifact V2 task instead.");
}
IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.PipelineArtifact, StringComparison.OrdinalIgnoreCase));
if (pipelineArtifacts.Count() == 0)
{
throw new ArgumentException("Could not find any pipeline artifacts in the build.");
}
else
{
context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count()));
var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
else if (downloadOptions == DownloadOptions.SingleDownload)
{
// 1) get manifest id from artifact data
BuildArtifact buildArtifact;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
if (buildArtifact.Resource.Type == PipelineArtifactConstants.FileShareArtifact)
{
throw new InvalidOperationException("File Share is not supported in the Download Pipeline Artifact V1 task. Please use Download Pipeline Artifact V2 task instead.");
}
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
}
}
}
// Download for version 2. This decision was made because version 1 is sealed and we didn't want to break any existing customers.
internal async Task DownloadAsyncV2(
AgentTaskPluginExecutionContext context,
PipelineArtifactDownloadParameters downloadParameters,
DownloadOptions downloadOptions,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BuildServer buildHelper = new BuildServer(connection);
// download all pipeline artifacts if artifact name is missing
if (downloadOptions == DownloadOptions.MultiDownload)
{
List <BuildArtifact> artifacts;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
artifacts = await buildHelper.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
IEnumerable <BuildArtifact> buildArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactConstants.Container);
IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactConstants.PipelineArtifact);
IEnumerable <BuildArtifact> fileShareArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactConstants.FileShareArtifact);
if (buildArtifacts.Any())
{
FileContainerProvider provider = new FileContainerProvider(connection, this.CreateTracer(context));
await provider.DownloadMultipleArtifactsAsync(downloadParameters, buildArtifacts, cancellationToken);
}
if (pipelineArtifacts.Any())
{
PipelineArtifactProvider provider = new PipelineArtifactProvider(context, connection, this.CreateTracer(context));
await provider.DownloadMultipleArtifactsAsync(downloadParameters, pipelineArtifacts, cancellationToken);
}
if (fileShareArtifacts.Any())
{
FileShareProvider provider = new FileShareProvider(context, connection, this.CreateTracer(context));
await provider.DownloadMultipleArtifactsAsync(downloadParameters, fileShareArtifacts, cancellationToken);
}
}
else if (downloadOptions == DownloadOptions.SingleDownload)
{
// 1) get manifest id from artifact data
BuildArtifact buildArtifact;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
ArtifactProviderFactory factory = new ArtifactProviderFactory(context, connection, this.CreateTracer(context));
IArtifactProvider provider = factory.GetProvider(buildArtifact);
await provider.DownloadSingleArtifactAsync(downloadParameters, buildArtifact, cancellationToken);
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
}
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint fingerprint,
string path,
CancellationToken cancellationToken,
string preferredContentFormat = ContentFormatConstants.Files)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
// Check if the key exists.
PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
//If cache exists, return.
if (getResult != null)
{
context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
return;
}
string uploadPath = await this.GetUploadPathAsync(preferredContentFormat, context, path, cancellationToken);
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken));
});
CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract
{
Fingerprint = fingerprint,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
ContentFormat = preferredContentFormat
};
// delete archive file if it's tar.
if (string.Equals(preferredContentFormat, ContentFormatConstants.SingleTar, StringComparison.OrdinalIgnoreCase))
{
try
{
if (File.Exists(uploadPath))
{
File.Delete(uploadPath);
}
}
catch { }
}
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
private async Task <string> GetUploadPathAsync(string preferredContentFormat, AgentTaskPluginExecutionContext context, string path, CancellationToken cancellationToken)
{
string uploadPath = path;
if (string.Equals(preferredContentFormat, ContentFormatConstants.SingleTar, StringComparison.OrdinalIgnoreCase))
{
uploadPath = await TarUtils.ArchiveFilesToTarAsync(context, path, cancellationToken);
}
return(uploadPath);
}
protected override async Task ProcessCommandInternalAsync(
AgentTaskPluginExecutionContext context,
CancellationToken token)
{
ArgUtil.NotNull(context, nameof(context));
string artifactName = context.GetInput(TaskProperties.ArtifactName, required: false);
string branchName = context.GetInput(TaskProperties.BranchName, required: false);
string definition = context.GetInput(TaskProperties.Definition, required: false);
string buildType = context.GetInput(TaskProperties.BuildType, required: true);
string specificBuildWithTriggering = context.GetInput(TaskProperties.SpecificBuildWithTriggering, required: false);
string buildVersionToDownload = context.GetInput(TaskProperties.BuildVersionToDownload, required: false);
string targetPath = context.GetInput(TaskProperties.DownloadPath, required: true);
string cleanDestinationFolder = context.GetInput(TaskProperties.CleanDestinationFolder, required: false);
string environmentBuildId = context.Variables.GetValueOrDefault(BuildVariables.BuildId)?.Value ?? string.Empty; // BuildID provided by environment.
string itemPattern = context.GetInput(TaskProperties.ItemPattern, required: false);
string projectName = context.GetInput(TaskProperties.Project, required: false);
string tags = context.GetInput(TaskProperties.Tags, required: false);
string allowPartiallySucceededBuilds = context.GetInput(TaskProperties.AllowPartiallySucceededBuilds, required: false);
string allowFailedBuilds = context.GetInput(TaskProperties.AllowFailedBuilds, required: false);
string allowCanceledBuilds = context.GetInput(TaskProperties.AllowCanceledBuilds, required: false);
string userSpecifiedBuildId = context.GetInput(TaskProperties.BuildId, required: false);
string defaultWorkingDirectory = context.Variables.GetValueOrDefault("system.defaultworkingdirectory").Value;
string downloadType = context.GetInput(TaskProperties.DownloadType, required: true);
// advanced
string retryDownloadCount = context.GetInput(TaskProperties.RetryDownloadCount, required: false);
string parallelizationLimit = context.GetInput(TaskProperties.ParallelizationLimit, required: false);
string checkDownloadedFiles = context.GetInput(TaskProperties.CheckDownloadedFiles, required: false);
string extractTars = context.GetInput(TaskProperties.ExtractTars, required: false);
string extractedTarsTempPath = Path.Combine(context.Variables.GetValueOrDefault("Agent.TempDirectory")?.Value, extractedTarsTempDir);
targetPath = Path.IsPathFullyQualified(targetPath) ? targetPath : Path.GetFullPath(Path.Combine(defaultWorkingDirectory, targetPath));
string[] minimatchPatterns = itemPattern.Split(
new[] { "\n" },
StringSplitOptions.RemoveEmptyEntries
);
string[] tagsInput = tags.Split(
new[] { "," },
StringSplitOptions.None
);
if (!bool.TryParse(allowPartiallySucceededBuilds, out var allowPartiallySucceededBuildsBool))
{
allowPartiallySucceededBuildsBool = false;
}
if (!bool.TryParse(allowFailedBuilds, out var allowFailedBuildsBool))
{
allowFailedBuildsBool = false;
}
if (!bool.TryParse(allowCanceledBuilds, out var allowCanceledBuildsBool))
{
allowCanceledBuildsBool = false;
}
if (!bool.TryParse(cleanDestinationFolder, out var cleanDestinationFolderBool))
{
cleanDestinationFolderBool = false;
}
var resultFilter = GetResultFilter(allowPartiallySucceededBuildsBool, allowFailedBuildsBool, allowCanceledBuildsBool);
if (!bool.TryParse(extractTars, out var extractTarsBool))
{
extractTarsBool = false;
}
if (extractTarsBool && PlatformUtil.RunningOnWindows)
{
throw new ArgumentException(StringUtil.Loc("TarExtractionNotSupportedInWindows"));
}
PipelineArtifactServer server = new PipelineArtifactServer(tracer);
ArtifactDownloadParameters downloadParameters;
if (buildType == buildTypeCurrent)
{
// TODO: use a constant for project id, which is currently defined in Microsoft.VisualStudio.Services.Agent.Constants.Variables.System.TeamProjectId (Ting)
string projectIdStr = context.Variables.GetValueOrDefault("system.teamProjectId")?.Value;
if (String.IsNullOrEmpty(projectIdStr))
{
throw new ArgumentNullException(StringUtil.Loc("CannotBeNullOrEmpty"), "Project ID");
}
Guid projectId = Guid.Parse(projectIdStr);
ArgUtil.NotEmpty(projectId, nameof(projectId));
int pipelineId = 0;
if (int.TryParse(environmentBuildId, out pipelineId) && pipelineId != 0)
{
OutputBuildInfo(context, pipelineId);
}
else
{
string hostType = context.Variables.GetValueOrDefault("system.hosttype")?.Value;
if (string.Equals(hostType, "Release", StringComparison.OrdinalIgnoreCase) ||
string.Equals(hostType, "DeploymentGroup", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("BuildIdIsNotAvailable", hostType ?? string.Empty, hostType ?? string.Empty));
}
else if (!string.Equals(hostType, "Build", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("CannotDownloadFromCurrentEnvironment", hostType ?? string.Empty));
}
else
{
// This should not happen since the build id comes from build environment. But a user may override that so we must be careful.
throw new ArgumentException(StringUtil.Loc("BuildIdIsNotValid", environmentBuildId));
}
}
downloadParameters = new ArtifactDownloadParameters
{
ProjectRetrievalOptions = BuildArtifactRetrievalOptions.RetrieveByProjectId,
ProjectId = projectId,
PipelineId = pipelineId,
ArtifactName = artifactName,
TargetDirectory = targetPath,
MinimatchFilters = minimatchPatterns,
MinimatchFilterWithArtifactName = true,
ParallelizationLimit = int.TryParse(parallelizationLimit, out var parallelLimit) ? parallelLimit : 8,
RetryDownloadCount = int.TryParse(retryDownloadCount, out var retryCount) ? retryCount : 4,
CheckDownloadedFiles = bool.TryParse(checkDownloadedFiles, out var checkDownloads) && checkDownloads,
CustomMinimatchOptions = minimatchOptions,
ExtractTars = extractTarsBool,
ExtractedTarsTempPath = extractedTarsTempPath
};
}
else if (buildType == buildTypeSpecific)
{
if (String.IsNullOrEmpty(projectName))
{
throw new ArgumentNullException(StringUtil.Loc("CannotBeNullOrEmpty"), "Project Name");
}
Guid projectId;
bool isProjGuid = Guid.TryParse(projectName, out projectId);
if (!isProjGuid)
{
projectId = await GetProjectIdAsync(context, projectName);
}
// Set the default pipelineId to 0, which is an invalid build id and it has to be reassigned to a valid build id.
int pipelineId = 0;
if (bool.TryParse(specificBuildWithTriggering, out var specificBuildWithTriggeringBool) && specificBuildWithTriggeringBool)
{
string hostType = context.Variables.GetValueOrDefault("system.hostType")?.Value;
string triggeringPipeline = null;
if (!string.IsNullOrWhiteSpace(hostType) && !hostType.Equals("build", StringComparison.OrdinalIgnoreCase)) // RM env.
{
var releaseAlias = context.Variables.GetValueOrDefault("release.triggeringartifact.alias")?.Value;
var definitionIdTriggered = context.Variables.GetValueOrDefault("release.artifacts." + releaseAlias ?? string.Empty + ".definitionId")?.Value;
if (!string.IsNullOrWhiteSpace(definitionIdTriggered) && definitionIdTriggered.Equals(definition, StringComparison.OrdinalIgnoreCase))
{
triggeringPipeline = context.Variables.GetValueOrDefault("release.artifacts." + releaseAlias ?? string.Empty + ".buildId")?.Value;
}
var triggeredProjectIdStr = context.Variables.GetValueOrDefault("release.artifacts." + releaseAlias + ".projectId")?.Value;
if (!string.IsNullOrWhiteSpace(triggeredProjectIdStr) && Guid.TryParse(triggeredProjectIdStr, out var triggeredProjectId))
{
projectId = triggeredProjectId;
}
}
else
{
var definitionIdTriggered = context.Variables.GetValueOrDefault("build.triggeredBy.definitionId")?.Value;
if (!string.IsNullOrWhiteSpace(definitionIdTriggered) && definitionIdTriggered.Equals(definition, StringComparison.OrdinalIgnoreCase))
{
triggeringPipeline = context.Variables.GetValueOrDefault("build.triggeredBy.buildId")?.Value;
}
var triggeredProjectIdStr = context.Variables.GetValueOrDefault("build.triggeredBy.projectId")?.Value;
if (!string.IsNullOrWhiteSpace(triggeredProjectIdStr) && Guid.TryParse(triggeredProjectIdStr, out var triggeredProjectId))
{
projectId = triggeredProjectId;
}
}
if (!string.IsNullOrWhiteSpace(triggeringPipeline))
{
pipelineId = int.Parse(triggeringPipeline);
}
}
if (pipelineId == 0)
{
if (buildVersionToDownload == buildVersionToDownloadLatest)
{
pipelineId = await this.GetPipelineIdAsync(context, definition, buildVersionToDownload, projectId.ToString(), tagsInput, resultFilter, null, cancellationToken : token);
}
else if (buildVersionToDownload == buildVersionToDownloadSpecific)
{
bool isPipelineIdNum = Int32.TryParse(userSpecifiedBuildId, out pipelineId);
if (!isPipelineIdNum)
{
throw new ArgumentException(StringUtil.Loc("RunIDNotValid", userSpecifiedBuildId));
}
}
else if (buildVersionToDownload == buildVersionToDownloadLatestFromBranch)
{
pipelineId = await this.GetPipelineIdAsync(context, definition, buildVersionToDownload, projectId.ToString(), tagsInput, resultFilter, branchName, cancellationToken : token);
}
else
{
throw new InvalidOperationException("Unreachable code!");
}
}
OutputBuildInfo(context, pipelineId);
downloadParameters = new ArtifactDownloadParameters
{
ProjectRetrievalOptions = BuildArtifactRetrievalOptions.RetrieveByProjectName,
ProjectName = projectName,
ProjectId = projectId,
PipelineId = pipelineId,
ArtifactName = artifactName,
TargetDirectory = targetPath,
MinimatchFilters = minimatchPatterns,
MinimatchFilterWithArtifactName = true,
ParallelizationLimit = int.TryParse(parallelizationLimit, out var parallelLimit) ? parallelLimit : 8,
RetryDownloadCount = int.TryParse(retryDownloadCount, out var retryCount) ? retryCount : 4,
CheckDownloadedFiles = bool.TryParse(checkDownloadedFiles, out var checkDownloads) && checkDownloads,
CustomMinimatchOptions = minimatchOptions,
ExtractTars = extractTarsBool,
ExtractedTarsTempPath = extractedTarsTempPath
};
}
else
{
throw new InvalidOperationException($"Build type '{buildType}' is not recognized.");
}
string fullPath = this.CreateDirectoryIfDoesntExist(targetPath);
if (cleanDestinationFolderBool)
{
CleanDirectory(context, fullPath);
}
var downloadOption = downloadType == "single" ? DownloadOptions.SingleDownload : DownloadOptions.MultiDownload;
// Build artifacts always includes the artifact in the path name
downloadParameters.IncludeArtifactNameInPath = true;
// By default, file container provider appends artifact name to target path when downloading specific files.
// This is undesirable because DownloadBuildArtifactsV0 doesn't do that.
// We also have a blob to enable appending artifact name just in case we break someone.
// By default, its value is going to be false, so we're defaulting to V0-like target path resolution.
downloadParameters.AppendArtifactNameToTargetPath =
AgentKnobs.EnableIncompatibleBuildArtifactsPathResolution.GetValue(context).AsBoolean();
context.Output(StringUtil.Loc("DownloadArtifactTo", targetPath));
await server.DownloadAsyncV2(context, downloadParameters, downloadOption, token);
context.Output(StringUtil.Loc("DownloadArtifactFinished"));
}
public Task RunAsync(AgentTaskPluginExecutionContext context, CancellationToken token)
{
this.tracer = new CallbackAppTraceSource(str => context.Output(str), System.Diagnostics.SourceLevels.Information);
return(this.ProcessCommandInternalAsync(context, token));
}
public async Task GetSourceAsync(
AgentTaskPluginExecutionContext executionContext,
Pipelines.RepositoryResource repository,
CancellationToken cancellationToken)
{
// Validate args.
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(repository, nameof(repository));
SvnCliManager svn = new SvnCliManager();
svn.Init(executionContext, repository, cancellationToken);
// Determine the sources directory.
string sourceDirectory = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Path);
executionContext.Debug($"sourceDirectory={sourceDirectory}");
ArgUtil.NotNullOrEmpty(sourceDirectory, nameof(sourceDirectory));
string sourceBranch = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Ref);
executionContext.Debug($"sourceBranch={sourceBranch}");
string revision = repository.Version;
if (string.IsNullOrWhiteSpace(revision))
{
revision = "HEAD";
}
executionContext.Debug($"revision={revision}");
bool clean = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Clean));
executionContext.Debug($"clean={clean}");
// Get the definition mappings.
var mappings = repository.Properties.Get <IList <Pipelines.WorkspaceMapping> >(Pipelines.RepositoryPropertyNames.Mappings);
List <SvnMappingDetails> allMappings = mappings.Select(x => new SvnMappingDetails()
{
ServerPath = x.ServerPath, LocalPath = x.LocalPath, Revision = x.Revision, Depth = x.Depth, IgnoreExternals = x.IgnoreExternals
}).ToList();
if (StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("system.debug")?.Value))
{
allMappings.ForEach(m => executionContext.Debug($"ServerPath: {m.ServerPath}, LocalPath: {m.LocalPath}, Depth: {m.Depth}, Revision: {m.Revision}, IgnoreExternals: {m.IgnoreExternals}"));
}
Dictionary <string, SvnMappingDetails> normalizedMappings = svn.NormalizeMappings(allMappings);
if (StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("system.debug")?.Value))
{
executionContext.Debug($"Normalized mappings count: {normalizedMappings.Count}");
normalizedMappings.ToList().ForEach(p => executionContext.Debug($" [{p.Key}] ServerPath: {p.Value.ServerPath}, LocalPath: {p.Value.LocalPath}, Depth: {p.Value.Depth}, Revision: {p.Value.Revision}, IgnoreExternals: {p.Value.IgnoreExternals}"));
}
string normalizedBranch = svn.NormalizeRelativePath(sourceBranch, '/', '\\');
executionContext.Output(StringUtil.Loc("SvnSyncingRepo", repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Name)));
string effectiveRevision = await svn.UpdateWorkspace(
sourceDirectory,
normalizedMappings,
clean,
normalizedBranch,
revision);
executionContext.Output(StringUtil.Loc("SvnBranchCheckedOut", normalizedBranch, repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Name), effectiveRevision));
}
public Task RunAsync(AgentTaskPluginExecutionContext context, CancellationToken token)
{
return(this.ProcessCommandInternalAsync(context, token));
}
public static int Main(string[] args)
{
Console.CancelKeyPress += Console_CancelKeyPress;
try
{
ArgUtil.NotNull(args, nameof(args));
ArgUtil.Equal(2, args.Length, nameof(args.Length));
string pluginType = args[0];
if (string.Equals("task", pluginType, StringComparison.OrdinalIgnoreCase))
{
string assemblyQualifiedName = args[1];
ArgUtil.NotNullOrEmpty(assemblyQualifiedName, nameof(assemblyQualifiedName));
// Set encoding to UTF8, process invoker will use UTF8 write to STDIN
Console.InputEncoding = Encoding.UTF8;
string serializedContext = Console.ReadLine();
ArgUtil.NotNullOrEmpty(serializedContext, nameof(serializedContext));
AgentTaskPluginExecutionContext executionContext = StringUtil.ConvertFromJson <AgentTaskPluginExecutionContext>(serializedContext);
ArgUtil.NotNull(executionContext, nameof(executionContext));
AssemblyLoadContext.Default.Resolving += ResolveAssembly;
try
{
Type type = Type.GetType(assemblyQualifiedName, throwOnError: true);
var taskPlugin = Activator.CreateInstance(type) as IAgentTaskPlugin;
ArgUtil.NotNull(taskPlugin, nameof(taskPlugin));
taskPlugin.RunAsync(executionContext, tokenSource.Token).GetAwaiter().GetResult();
}
catch (Exception ex)
{
// any exception throw from plugin will fail the task.
executionContext.Error(ex.ToString());
}
finally
{
AssemblyLoadContext.Default.Resolving -= ResolveAssembly;
}
return(0);
}
else if (string.Equals("command", pluginType, StringComparison.OrdinalIgnoreCase))
{
string assemblyQualifiedName = args[1];
ArgUtil.NotNullOrEmpty(assemblyQualifiedName, nameof(assemblyQualifiedName));
string serializedContext = Console.ReadLine();
ArgUtil.NotNullOrEmpty(serializedContext, nameof(serializedContext));
AgentCommandPluginExecutionContext executionContext = StringUtil.ConvertFromJson <AgentCommandPluginExecutionContext>(serializedContext);
ArgUtil.NotNull(executionContext, nameof(executionContext));
AssemblyLoadContext.Default.Resolving += ResolveAssembly;
try
{
Type type = Type.GetType(assemblyQualifiedName, throwOnError: true);
var commandPlugin = Activator.CreateInstance(type) as IAgentCommandPlugin;
ArgUtil.NotNull(commandPlugin, nameof(commandPlugin));
commandPlugin.ProcessCommandAsync(executionContext, tokenSource.Token).GetAwaiter().GetResult();
}
catch (Exception ex)
{
// any exception throw from plugin will fail the command.
executionContext.Error(ex.ToString());
}
finally
{
AssemblyLoadContext.Default.Resolving -= ResolveAssembly;
}
return(0);
}
else
{
throw new ArgumentOutOfRangeException(pluginType);
}
}
catch (Exception ex)
{
// infrastructure failure.
Console.Error.WriteLine(ex.ToString());
return(1);
}
finally
{
Console.CancelKeyPress -= Console_CancelKeyPress;
}
}
public async Task GetSourceAsync(
AgentTaskPluginExecutionContext executionContext,
Pipelines.RepositoryResource repository,
CancellationToken cancellationToken)
{
// Validate args.
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(repository, nameof(repository));
#if OS_WINDOWS
// Validate .NET Framework 4.6 or higher is installed.
if (!NetFrameworkUtil.Test(new Version(4, 6), executionContext))
{
throw new Exception(StringUtil.Loc("MinimumNetFramework46"));
}
#endif
// Create the tf command manager.
#if OS_WINDOWS
var tf = new TFCliManager();
#else
var tf = new TeeCliManager();
#endif
tf.CancellationToken = cancellationToken;
tf.Repository = repository;
tf.ExecutionContext = executionContext;
if (repository.Endpoint != null)
{
// the endpoint should either be the SystemVssConnection (id = guild.empty, name = SystemVssConnection)
// or a real service endpoint to external service which has a real id
var endpoint = executionContext.Endpoints.Single(
x => (repository.Endpoint.Id != Guid.Empty && x.Id == repository.Endpoint.Id) ||
(repository.Endpoint.Id == Guid.Empty && string.Equals(x.Name, repository.Endpoint.Name.ToString(), StringComparison.OrdinalIgnoreCase)));
ArgUtil.NotNull(endpoint, nameof(endpoint));
tf.Endpoint = endpoint;
}
// Setup proxy.
var agentProxy = executionContext.GetProxyConfiguration();
if (agentProxy != null && !string.IsNullOrEmpty(agentProxy.ProxyAddress) && !agentProxy.WebProxy.IsBypassed(repository.Url))
{
executionContext.Debug($"Configure '{tf.FilePath}' to work through proxy server '{agentProxy.ProxyAddress}'.");
tf.SetupProxy(agentProxy.ProxyAddress, agentProxy.ProxyUsername, agentProxy.ProxyPassword);
}
// Setup client certificate.
var agentCertManager = executionContext.GetCertConfiguration();
if (agentCertManager != null && agentCertManager.SkipServerCertificateValidation)
{
#if OS_WINDOWS
executionContext.Debug("TF.exe does not support ignore SSL certificate validation error.");
#else
executionContext.Debug("TF does not support ignore SSL certificate validation error.");
#endif
}
// prepare client cert, if the repository's endpoint url match the TFS/VSTS url
var systemConnection = executionContext.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrEmpty(agentCertManager?.ClientCertificateFile) &&
Uri.Compare(repository.Url, systemConnection.Url, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) == 0)
{
executionContext.Debug($"Configure '{tf.FilePath}' to work with client cert '{agentCertManager.ClientCertificateFile}'.");
tf.SetupClientCertificate(agentCertManager.ClientCertificateFile, agentCertManager.ClientCertificatePrivateKeyFile, agentCertManager.ClientCertificateArchiveFile, agentCertManager.ClientCertificatePassword);
}
// Add TF to the PATH.
string tfPath = tf.FilePath;
ArgUtil.File(tfPath, nameof(tfPath));
executionContext.Output(StringUtil.Loc("Prepending0WithDirectoryContaining1", PathUtil.PathVariable, Path.GetFileName(tfPath)));
executionContext.PrependPath(Path.GetDirectoryName(tfPath));
executionContext.Debug($"PATH: '{Environment.GetEnvironmentVariable("PATH")}'");
#if OS_WINDOWS
// Set TFVC_BUILDAGENT_POLICYPATH
string policyDllPath = Path.Combine(executionContext.Variables.GetValueOrDefault("Agent.ServerOMDirectory")?.Value, "Microsoft.TeamFoundation.VersionControl.Controls.dll");
ArgUtil.File(policyDllPath, nameof(policyDllPath));
const string policyPathEnvKey = "TFVC_BUILDAGENT_POLICYPATH";
executionContext.Output(StringUtil.Loc("SetEnvVar", policyPathEnvKey));
executionContext.SetVariable(policyPathEnvKey, policyDllPath);
#endif
// Check if the administrator accepted the license terms of the TEE EULA when configuring the agent.
if (tf.Features.HasFlag(TfsVCFeatures.Eula) && StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("Agent.AcceptTeeEula")?.Value))
{
// Check if the "tf eula -accept" command needs to be run for the current user.
bool skipEula = false;
try
{
skipEula = tf.TestEulaAccepted();
}
catch (Exception ex)
{
executionContext.Debug("Unexpected exception while testing whether the TEE EULA has been accepted for the current user.");
executionContext.Debug(ex.ToString());
}
if (!skipEula)
{
// Run the command "tf eula -accept".
try
{
await tf.EulaAsync();
}
catch (Exception ex)
{
executionContext.Debug(ex.ToString());
executionContext.Warning(ex.Message);
}
}
}
// Get the workspaces.
executionContext.Output(StringUtil.Loc("QueryingWorkspaceInfo"));
ITfsVCWorkspace[] tfWorkspaces = await tf.WorkspacesAsync();
// Determine the workspace name.
string buildDirectory = executionContext.Variables.GetValueOrDefault("agent.builddirectory")?.Value;
ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory));
string workspaceName = $"ws_{Path.GetFileName(buildDirectory)}_{executionContext.Variables.GetValueOrDefault("agent.id")?.Value}";
executionContext.SetVariable("build.repository.tfvc.workspace", workspaceName);
// Get the definition mappings.
var workspaceMappings = repository.Properties.Get <IList <Pipelines.WorkspaceMapping> >(Pipelines.RepositoryPropertyNames.Mappings);
DefinitionWorkspaceMapping[] definitionMappings = workspaceMappings.Select(x => new DefinitionWorkspaceMapping()
{
ServerPath = x.ServerPath, LocalPath = x.LocalPath, MappingType = x.Exclude ? DefinitionMappingType.Cloak : DefinitionMappingType.Map
}).ToArray();
// Determine the sources directory.
string sourcesDirectory = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Path);
ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory));
// Attempt to re-use an existing workspace if the command manager supports scorch
// or if clean is not specified.
ITfsVCWorkspace existingTFWorkspace = null;
bool clean = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Clean));
if (tf.Features.HasFlag(TfsVCFeatures.Scorch) || !clean)
{
existingTFWorkspace = WorkspaceUtil.MatchExactWorkspace(
executionContext: executionContext,
tfWorkspaces: tfWorkspaces,
name: workspaceName,
definitionMappings: definitionMappings,
sourcesDirectory: sourcesDirectory);
if (existingTFWorkspace != null)
{
if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot))
{
// Undo pending changes.
ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory);
if (tfStatus?.HasPendingChanges ?? false)
{
await tf.UndoAsync(localPath : sourcesDirectory);
// Cleanup remaining files/directories from pend adds.
tfStatus.AllAdds
.OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted.
.ToList()
.ForEach(x =>
{
executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem));
IOUtil.Delete(x.LocalItem, cancellationToken);
});
}
}
else
{
// Perform "undo" for each map.
foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0])
{
if (definitionMapping.MappingType == DefinitionMappingType.Map)
{
// Check the status.
string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory);
ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath);
if (tfStatus?.HasPendingChanges ?? false)
{
// Undo.
await tf.UndoAsync(localPath : localPath);
// Cleanup remaining files/directories from pend adds.
tfStatus.AllAdds
.OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted.
.ToList()
.ForEach(x =>
{
executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem));
IOUtil.Delete(x.LocalItem, cancellationToken);
});
}
}
}
}
// Scorch.
if (clean)
{
// Try to scorch.
try
{
await tf.ScorchAsync();
}
catch (ProcessExitCodeException ex)
{
// Scorch failed.
// Warn, drop the folder, and re-clone.
executionContext.Warning(ex.Message);
existingTFWorkspace = null;
}
}
}
}
// Create a new workspace.
if (existingTFWorkspace == null)
{
// Remove any conflicting workspaces.
await RemoveConflictingWorkspacesAsync(
tf : tf,
tfWorkspaces : tfWorkspaces,
name : workspaceName,
directory : sourcesDirectory);
// Remove any conflicting workspace from a different computer.
// This is primarily a hosted scenario where a registered hosted
// agent can land on a different computer each time.
tfWorkspaces = await tf.WorkspacesAsync(matchWorkspaceNameOnAnyComputer : true);
foreach (ITfsVCWorkspace tfWorkspace in tfWorkspaces ?? new ITfsVCWorkspace[0])
{
await tf.WorkspaceDeleteAsync(tfWorkspace);
}
// Recreate the sources directory.
executionContext.Debug($"Deleting: '{sourcesDirectory}'.");
IOUtil.DeleteDirectory(sourcesDirectory, cancellationToken);
Directory.CreateDirectory(sourcesDirectory);
// Create the workspace.
await tf.WorkspaceNewAsync();
// Remove the default mapping.
if (tf.Features.HasFlag(TfsVCFeatures.DefaultWorkfoldMap))
{
await tf.WorkfoldUnmapAsync("$/");
}
// Sort the definition mappings.
definitionMappings =
(definitionMappings ?? new DefinitionWorkspaceMapping[0])
.OrderBy(x => x.NormalizedServerPath?.Length ?? 0) // By server path length.
.ToArray() ?? new DefinitionWorkspaceMapping[0];
// Add the definition mappings to the workspace.
foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings)
{
switch (definitionMapping.MappingType)
{
case DefinitionMappingType.Cloak:
// Add the cloak.
await tf.WorkfoldCloakAsync(serverPath : definitionMapping.ServerPath);
break;
case DefinitionMappingType.Map:
// Add the mapping.
await tf.WorkfoldMapAsync(
serverPath : definitionMapping.ServerPath,
localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory));
break;
default:
throw new NotSupportedException();
}
}
}
if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot))
{
// Get.
await tf.GetAsync(localPath : sourcesDirectory);
}
else
{
// Perform "get" for each map.
foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0])
{
if (definitionMapping.MappingType == DefinitionMappingType.Map)
{
await tf.GetAsync(localPath : definitionMapping.GetRootedLocalPath(sourcesDirectory));
}
}
}
// Steps for shelveset/gated.
string shelvesetName = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Shelveset);
if (!string.IsNullOrEmpty(shelvesetName))
{
// Steps for gated.
ITfsVCShelveset tfShelveset = null;
string gatedShelvesetName = executionContext.Variables.GetValueOrDefault("build.gated.shelvesetname")?.Value;
if (!string.IsNullOrEmpty(gatedShelvesetName))
{
// Clean the last-saved-checkin-metadata for existing workspaces.
//
// A better long term fix is to add a switch to "tf unshelve" that completely overwrites
// the last-saved-checkin-metadata, instead of merging associated work items.
//
// The targeted workaround for now is to create a trivial change and "tf shelve /move",
// which will delete the last-saved-checkin-metadata.
if (existingTFWorkspace != null)
{
executionContext.Output("Cleaning last saved checkin metadata.");
// Find a local mapped directory.
string firstLocalDirectory =
(definitionMappings ?? new DefinitionWorkspaceMapping[0])
.Where(x => x.MappingType == DefinitionMappingType.Map)
.Select(x => x.GetRootedLocalPath(sourcesDirectory))
.FirstOrDefault(x => Directory.Exists(x));
if (firstLocalDirectory == null)
{
executionContext.Warning("No mapped folder found. Unable to clean last-saved-checkin-metadata.");
}
else
{
// Create a trival change and "tf shelve /move" to clear the
// last-saved-checkin-metadata.
string cleanName = "__tf_clean_wksp_metadata";
string tempCleanFile = Path.Combine(firstLocalDirectory, cleanName);
try
{
File.WriteAllText(path: tempCleanFile, contents: "clean last-saved-checkin-metadata", encoding: Encoding.UTF8);
await tf.AddAsync(tempCleanFile);
await tf.ShelveAsync(shelveset : cleanName, commentFile : tempCleanFile, move : true);
}
catch (Exception ex)
{
executionContext.Warning($"Unable to clean last-saved-checkin-metadata. {ex.Message}");
try
{
await tf.UndoAsync(tempCleanFile);
}
catch (Exception ex2)
{
executionContext.Warning($"Unable to undo '{tempCleanFile}'. {ex2.Message}");
}
}
finally
{
IOUtil.DeleteFile(tempCleanFile);
}
}
}
// Get the shelveset metadata.
tfShelveset = await tf.ShelvesetsAsync(shelveset : shelvesetName);
// The above command throws if the shelveset is not found,
// so the following assertion should never fail.
ArgUtil.NotNull(tfShelveset, nameof(tfShelveset));
}
// Unshelve.
await tf.UnshelveAsync(shelveset : shelvesetName);
// Ensure we undo pending changes for shelveset build at the end.
executionContext.SetTaskVariable("UndoShelvesetPendingChanges", bool.TrueString);
if (!string.IsNullOrEmpty(gatedShelvesetName))
{
// Create the comment file for reshelve.
StringBuilder comment = new StringBuilder(tfShelveset.Comment ?? string.Empty);
string runCi = executionContext.Variables.GetValueOrDefault("build.gated.runci")?.Value;
bool gatedRunCi = StringUtil.ConvertToBoolean(runCi, true);
if (!gatedRunCi)
{
if (comment.Length > 0)
{
comment.AppendLine();
}
comment.Append("***NO_CI***");
}
string commentFile = null;
try
{
commentFile = Path.GetTempFileName();
File.WriteAllText(path: commentFile, contents: comment.ToString(), encoding: Encoding.UTF8);
// Reshelve.
await tf.ShelveAsync(shelveset : gatedShelvesetName, commentFile : commentFile, move : false);
}
finally
{
// Cleanup the comment file.
if (File.Exists(commentFile))
{
File.Delete(commentFile);
}
}
}
}
// Cleanup proxy settings.
if (agentProxy != null && !string.IsNullOrEmpty(agentProxy.ProxyAddress) && !agentProxy.WebProxy.IsBypassed(repository.Url))
{
executionContext.Debug($"Remove proxy setting for '{tf.FilePath}' to work through proxy server '{agentProxy.ProxyAddress}'.");
tf.CleanupProxySetting();
}
// Set intra-task variable for post job cleanup
executionContext.SetTaskVariable("repository", repository.Alias);
}
public Task RunAsync(AgentTaskPluginExecutionContext context, CancellationToken token)
{
this.tracer = context.CreateArtifactsTracer();
return(this.ProcessCommandInternalAsync(context, token));
}
public async Task PostJobCleanupAsync(AgentTaskPluginExecutionContext executionContext, Pipelines.RepositoryResource repository)
{
bool undoShelvesetPendingChanges = StringUtil.ConvertToBoolean(executionContext.TaskVariables.GetValueOrDefault("UndoShelvesetPendingChanges")?.Value);
if (undoShelvesetPendingChanges)
{
string shelvesetName = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Shelveset);
executionContext.Debug($"Undo pending changes left by shelveset '{shelvesetName}'.");
// Create the tf command manager.
#if OS_WINDOWS
var tf = new TFCliManager();
#else
var tf = new TeeCliManager();
#endif
tf.CancellationToken = CancellationToken.None;
tf.Repository = repository;
tf.ExecutionContext = executionContext;
if (repository.Endpoint != null)
{
// the endpoint should either be the SystemVssConnection (id = guild.empty, name = SystemVssConnection)
// or a real service endpoint to external service which has a real id
var endpoint = executionContext.Endpoints.Single(
x => (repository.Endpoint.Id != Guid.Empty && x.Id == repository.Endpoint.Id) ||
(repository.Endpoint.Id == Guid.Empty && string.Equals(x.Name, repository.Endpoint.Name.ToString(), StringComparison.OrdinalIgnoreCase)));
ArgUtil.NotNull(endpoint, nameof(endpoint));
tf.Endpoint = endpoint;
}
// Get the definition mappings.
var workspaceMappings = repository.Properties.Get <IList <Pipelines.WorkspaceMapping> >(Pipelines.RepositoryPropertyNames.Mappings);
DefinitionWorkspaceMapping[] definitionMappings = workspaceMappings.Select(x => new DefinitionWorkspaceMapping()
{
ServerPath = x.ServerPath, LocalPath = x.LocalPath, MappingType = x.Exclude ? DefinitionMappingType.Cloak : DefinitionMappingType.Map
}).ToArray();
// Determine the sources directory.
string sourcesDirectory = repository.Properties.Get <string>(Pipelines.RepositoryPropertyNames.Path);
ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory));
try
{
if (tf.Features.HasFlag(TfsVCFeatures.GetFromUnmappedRoot))
{
// Undo pending changes.
ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : sourcesDirectory);
if (tfStatus?.HasPendingChanges ?? false)
{
await tf.UndoAsync(localPath : sourcesDirectory);
// Cleanup remaining files/directories from pend adds.
tfStatus.AllAdds
.OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted.
.ToList()
.ForEach(x =>
{
executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem));
IOUtil.Delete(x.LocalItem, CancellationToken.None);
});
}
}
else
{
// Perform "undo" for each map.
foreach (DefinitionWorkspaceMapping definitionMapping in definitionMappings ?? new DefinitionWorkspaceMapping[0])
{
if (definitionMapping.MappingType == DefinitionMappingType.Map)
{
// Check the status.
string localPath = definitionMapping.GetRootedLocalPath(sourcesDirectory);
ITfsVCStatus tfStatus = await tf.StatusAsync(localPath : localPath);
if (tfStatus?.HasPendingChanges ?? false)
{
// Undo.
await tf.UndoAsync(localPath : localPath);
// Cleanup remaining files/directories from pend adds.
tfStatus.AllAdds
.OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted.
.ToList()
.ForEach(x =>
{
executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem));
IOUtil.Delete(x.LocalItem, CancellationToken.None);
});
}
}
}
}
}
catch (Exception ex)
{
// We can't undo pending changes, log a warning and continue.
executionContext.Debug(ex.ToString());
executionContext.Warning(ex.Message);
}
}
}
private void OutputBuildInfo(AgentTaskPluginExecutionContext context, int?pipelineId)
{
context.Output(StringUtil.Loc("DownloadingFromBuild", pipelineId));
// populate output variable 'BuildNumber' with buildId
context.SetVariable("BuildNumber", pipelineId.ToString());
}
public static ITfsVCWorkspace MatchExactWorkspace(
AgentTaskPluginExecutionContext executionContext,
ITfsVCWorkspace[] tfWorkspaces,
string name,
DefinitionWorkspaceMapping[] definitionMappings,
string sourcesDirectory)
{
ArgUtil.NotNullOrEmpty(name, nameof(name));
ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory));
// Short-circuit early if the sources directory is empty.
//
// Consider the sources directory to be empty if it only contains a .tf directory exists. This can
// indicate the workspace is in a corrupted state and the tf commands (e.g. status) will not return
// reliable information. An easy way to reproduce this is to delete the workspace directory, then
// run "tf status" on that workspace. The .tf directory will be recreated but the contents will be
// in a corrupted state.
if (!Directory.Exists(sourcesDirectory) ||
!Directory.EnumerateFileSystemEntries(sourcesDirectory).Any(x => !x.EndsWith($"{Path.DirectorySeparatorChar}.tf")))
{
executionContext.Debug("Sources directory does not exist or is empty.");
return(null);
}
string machineName = Environment.MachineName;
executionContext.Debug($"Attempting to find a workspace: '{name}'");
foreach (ITfsVCWorkspace tfWorkspace in tfWorkspaces ?? new ITfsVCWorkspace[0])
{
// Compare the workspace name.
if (!string.Equals(tfWorkspace.Name, name, StringComparison.Ordinal))
{
executionContext.Debug($"Skipping workspace: '{tfWorkspace.Name}'");
continue;
}
executionContext.Debug($"Candidate workspace: '{tfWorkspace.Name}'");
// Compare the machine name.
if (!string.Equals(tfWorkspace.Computer, machineName, StringComparison.Ordinal))
{
executionContext.Debug($"Expected computer name: '{machineName}'. Actual: '{tfWorkspace.Computer}'");
continue;
}
// Compare the number of mappings.
if ((tfWorkspace.Mappings?.Length ?? 0) != (definitionMappings?.Length ?? 0))
{
executionContext.Debug($"Expected number of mappings: '{definitionMappings?.Length ?? 0}'. Actual: '{tfWorkspace.Mappings?.Length ?? 0}'");
continue;
}
// Sort the definition mappings.
List <DefinitionWorkspaceMapping> sortedDefinitionMappings =
(definitionMappings ?? new DefinitionWorkspaceMapping[0])
.OrderBy(x => x.MappingType != DefinitionMappingType.Cloak) // Cloaks first
.ThenBy(x => !x.Recursive) // Then recursive maps
.ThenBy(x => x.NormalizedServerPath) // Then sort by the normalized server path
.ToList();
for (int i = 0; i < sortedDefinitionMappings.Count; i++)
{
DefinitionWorkspaceMapping mapping = sortedDefinitionMappings[i];
executionContext.Debug($"Definition mapping[{i}]: cloak '{mapping.MappingType == DefinitionMappingType.Cloak}', recursive '{mapping.Recursive}', server path '{mapping.NormalizedServerPath}', local path '{mapping.GetRootedLocalPath(sourcesDirectory)}'");
}
// Sort the TF mappings.
List <ITfsVCMapping> sortedTFMappings =
(tfWorkspace.Mappings ?? new ITfsVCMapping[0])
.OrderBy(x => !x.Cloak) // Cloaks first
.ThenBy(x => !x.Recursive) // Then recursive maps
.ThenBy(x => x.ServerPath) // Then sort by server path
.ToList();
for (int i = 0; i < sortedTFMappings.Count; i++)
{
ITfsVCMapping mapping = sortedTFMappings[i];
executionContext.Debug($"Found mapping[{i}]: cloak '{mapping.Cloak}', recursive '{mapping.Recursive}', server path '{mapping.ServerPath}', local path '{mapping.LocalPath}'");
}
// Compare the mappings.
bool allMatch = true;
List <string> matchTrace = new List <string>();
for (int i = 0; i < sortedTFMappings.Count; i++)
{
ITfsVCMapping tfMapping = sortedTFMappings[i];
DefinitionWorkspaceMapping definitionMapping = sortedDefinitionMappings[i];
// Compare the cloak flag.
bool expectedCloak = definitionMapping.MappingType == DefinitionMappingType.Cloak;
if (tfMapping.Cloak != expectedCloak)
{
matchTrace.Add(StringUtil.Loc("ExpectedMappingCloak", i, expectedCloak, tfMapping.Cloak));
allMatch = false;
break;
}
// Compare the recursive flag.
if (!expectedCloak && tfMapping.Recursive != definitionMapping.Recursive)
{
matchTrace.Add(StringUtil.Loc("ExpectedMappingRecursive", i, definitionMapping.Recursive, tfMapping.Recursive));
allMatch = false;
break;
}
// Compare the server path. Normalize the expected server path for a single-level map.
string expectedServerPath = definitionMapping.NormalizedServerPath;
if (!string.Equals(tfMapping.ServerPath, expectedServerPath, StringComparison.Ordinal))
{
matchTrace.Add(StringUtil.Loc("ExpectedMappingServerPath", i, expectedServerPath, tfMapping.ServerPath));
allMatch = false;
break;
}
// Compare the local path.
if (!expectedCloak)
{
string expectedLocalPath = definitionMapping.GetRootedLocalPath(sourcesDirectory);
if (!string.Equals(tfMapping.LocalPath, expectedLocalPath, StringComparison.Ordinal))
{
matchTrace.Add(StringUtil.Loc("ExpectedMappingLocalPath", i, expectedLocalPath, tfMapping.LocalPath));
allMatch = false;
break;
}
}
}
if (allMatch)
{
executionContext.Debug("Matching workspace found.");
return(tfWorkspace);
}
else
{
executionContext.Output(StringUtil.Loc("WorkspaceMappingNotMatched", tfWorkspace.Name));
foreach (var trace in matchTrace)
{
executionContext.Output(trace);
}
}
}
executionContext.Debug("Matching workspace not found.");
return(null);
}
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint[] fingerprints,
string path,
string cacheHitVariable,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
if (result != null)
{
context.Output($"Entry found at fingerprint: `{result.Fingerprint.ToString()}`");
context.Verbose($"Manifest ID is: {result.ManifestId.ValueString}");
PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, path, Enum.Parse <ContentFormat>(result.ContentFormat), cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord);
context.Output("Cache restored.");
}
if (!string.IsNullOrEmpty(cacheHitVariable))
{
if (result == null)
{
context.SetVariable(cacheHitVariable, "false");
}
else
{
context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`");
bool foundExact = false;
foreach (var fingerprint in fingerprints)
{
context.Verbose($"This fingerprint: `{fingerprint.ToString()}`");
if (fingerprint == result.Fingerprint ||
result.Fingerprint.Segments.Length == 1 && result.Fingerprint.Segments.Single() == fingerprint.SummarizeForV1())
{
foundExact = true;
break;
}
}
context.SetVariable(cacheHitVariable, foundExact ? "true" : "inexact");
}
}
}
}
public async Task RunPluginTaskAsync(IExecutionContext context, string plugin, Dictionary <string, string> inputs, Dictionary <string, string> environment, Variables runtimeVariables, EventHandler <ProcessDataReceivedEventArgs> outputHandler)
{
ArgUtil.NotNullOrEmpty(plugin, nameof(plugin));
// Only allow plugins we defined
if (!_taskPlugins.Contains(plugin))
{
throw new NotSupportedException(plugin);
}
// Resolve the working directory.
string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
ArgUtil.Directory(workingDirectory, nameof(workingDirectory));
// Agent.PluginHost
string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
ArgUtil.File(file, $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
// Agent.PluginHost's arguments
string arguments = $"task \"{plugin}\"";
// construct plugin context
var target = context.StepTarget();
Variables.TranslationMethod translateToHostPath = Variables.DefaultStringTranslator;
ContainerInfo containerInfo = target as ContainerInfo;
// Since plugins run on the host, but the inputs and variables have already been translated
// to the container path, we need to convert them back to the host path
// TODO: look to see if there is a better way to not have translate these back
if (containerInfo != null)
{
var newInputs = new Dictionary <string, string>();
foreach (var entry in inputs)
{
newInputs[entry.Key] = containerInfo.TranslateToHostPath(entry.Value);
}
inputs = newInputs;
translateToHostPath = (string val) => { return(containerInfo.TranslateToHostPath(val)); };
}
AgentTaskPluginExecutionContext pluginContext = new AgentTaskPluginExecutionContext
{
Inputs = inputs,
Repositories = context.Repositories,
Endpoints = context.Endpoints,
Container = containerInfo, //TODO: Figure out if this needs to have all the containers or just the one for the current step
JobSettings = context.JobSettings,
};
// variables
runtimeVariables.CopyInto(pluginContext.Variables, translateToHostPath);
context.TaskVariables.CopyInto(pluginContext.TaskVariables, translateToHostPath);
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
var redirectStandardIn = new InputQueue <string>();
redirectStandardIn.Enqueue(JsonUtility.ToString(pluginContext));
processInvoker.OutputDataReceived += outputHandler;
processInvoker.ErrorDataReceived += outputHandler;
// Execute the process. Exit code 0 should always be returned.
// A non-zero exit code indicates infrastructural failure.
// Task failure should be communicated over STDOUT using ## commands.
await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : file,
arguments : arguments,
environment : environment,
requireExitCodeZero : true,
outputEncoding : Encoding.UTF8,
killProcessOnCancel : false,
redirectStandardIn : redirectStandardIn,
cancellationToken : context.CancellationToken);
}
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint fingerprint,
string path,
CancellationToken cancellationToken,
ContentFormat contentFormat)
{
VssConnection connection = context.VssConnection;
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
// Check if the key exists.
PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
//If cache exists, return.
if (getResult != null)
{
context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
return;
}
string uploadPath = await this.GetUploadPathAsync(contentFormat, context, path, cancellationToken);
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract
{
Fingerprint = fingerprint,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
ContentFormat = contentFormat.ToString(),
};
// delete archive file if it's tar.
if (contentFormat == ContentFormat.SingleTar)
{
try
{
if (File.Exists(uploadPath))
{
File.Delete(uploadPath);
}
}
catch { }
}
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
public async Task RunPluginTaskAsync(IExecutionContext context, string plugin, Dictionary <string, string> inputs, Dictionary <string, string> environment, Variables runtimeVariables, EventHandler <ProcessDataReceivedEventArgs> outputHandler)
{
ArgUtil.NotNullOrEmpty(plugin, nameof(plugin));
// Only allow plugins we defined
if (!_taskPlugins.Contains(plugin))
{
throw new NotSupportedException(plugin);
}
// Resolve the working directory.
string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
ArgUtil.Directory(workingDirectory, nameof(workingDirectory));
// Agent.PluginHost
string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
ArgUtil.File(file, $"Agent.PluginHost{Util.IOUtil.ExeExtension}");
// Agent.PluginHost's arguments
string arguments = $"task \"{plugin}\"";
// construct plugin context
AgentTaskPluginExecutionContext pluginContext = new AgentTaskPluginExecutionContext
{
Inputs = inputs,
Repositories = context.Repositories,
Endpoints = context.Endpoints
};
// variables
foreach (var publicVar in runtimeVariables.Public)
{
pluginContext.Variables[publicVar.Key] = publicVar.Value;
}
foreach (var publicVar in runtimeVariables.Private)
{
pluginContext.Variables[publicVar.Key] = new VariableValue(publicVar.Value, true);
}
// task variables (used by wrapper task)
foreach (var publicVar in context.TaskVariables.Public)
{
pluginContext.TaskVariables[publicVar.Key] = publicVar.Value;
}
foreach (var publicVar in context.TaskVariables.Private)
{
pluginContext.TaskVariables[publicVar.Key] = new VariableValue(publicVar.Value, true);
}
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
var redirectStandardIn = new InputQueue <string>();
redirectStandardIn.Enqueue(JsonUtility.ToString(pluginContext));
processInvoker.OutputDataReceived += outputHandler;
processInvoker.ErrorDataReceived += outputHandler;
// Execute the process. Exit code 0 should always be returned.
// A non-zero exit code indicates infrastructural failure.
// Task failure should be communicated over STDOUT using ## commands.
await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : file,
arguments : arguments,
environment : environment,
requireExitCodeZero : true,
outputEncoding : Encoding.UTF8,
killProcessOnCancel : false,
redirectStandardIn : redirectStandardIn,
cancellationToken : context.CancellationToken);
}
}
protected override string GetArtifactName(AgentTaskPluginExecutionContext context)
{
return(context.GetInput(ArtifactEventProperties.ArtifactName, required: false));
}
public override Task <Version> GitLfsVersion(AgentTaskPluginExecutionContext context)
{
return(Task.FromResult(new Version("2.30.2")));
}
