public async Task <int> LocalRunAsync(CommandSettings command, CancellationToken token)
{
Trace.Info(nameof(LocalRunAsync));
// Warn preview.
_term.WriteLine("This command is currently in preview. The interface and behavior will change in a future version.");
if (!command.Unattended)
{
_term.WriteLine("Press Enter to continue.");
_term.ReadLine();
}
HostContext.RunMode = RunMode.Local;
// Resolve the YAML file path.
string ymlFile = command.GetYml();
if (string.IsNullOrEmpty(ymlFile))
{
string[] ymlFiles =
Directory.GetFiles(Directory.GetCurrentDirectory())
.Where((string filePath) =>
{
return(filePath.EndsWith(".yml", IOUtil.FilePathStringComparison));
})
.ToArray();
if (ymlFiles.Length > 1)
{
throw new Exception($"More than one .yml file exists in the current directory. Specify which file to use via the '{Constants.Agent.CommandLine.Args.Yml}' command line argument.");
}
ymlFile = ymlFiles.FirstOrDefault();
}
if (string.IsNullOrEmpty(ymlFile))
{
throw new Exception($"Unable to find a .yml file in the current directory. Specify which file to use via the '{Constants.Agent.CommandLine.Args.Yml}' command line argument.");
}
// Load the YAML file.
var parseOptions = new ParseOptions
{
MaxFiles = 10,
MustacheEvaluationMaxResultLength = 512 * 1024, // 512k string length
MustacheEvaluationTimeout = TimeSpan.FromSeconds(10),
MustacheMaxDepth = 5,
};
var pipelineParser = new PipelineParser(new PipelineTraceWriter(), new PipelineFileProvider(), parseOptions);
Pipelines.Process process = pipelineParser.Load(
defaultRoot: Directory.GetCurrentDirectory(),
path: ymlFile,
mustacheContext: null,
cancellationToken: HostContext.AgentShutdownToken);
ArgUtil.NotNull(process, nameof(process));
if (command.WhatIf)
{
return(Constants.Agent.ReturnCode.Success);
}
// Verify the current directory is the root of a git repo.
string repoDirectory = Directory.GetCurrentDirectory();
if (!Directory.Exists(Path.Combine(repoDirectory, ".git")))
{
throw new Exception("Unable to run the build locally. The command must be executed from the root directory of a local git repository.");
}
// Get the URL - required if missing tasks.
string url = command.GetUrl(suppressPromptIfEmpty: true);
if (string.IsNullOrEmpty(url))
{
if (!TestAllTasksCached(process, token))
{
url = command.GetUrl(suppressPromptIfEmpty: false);
}
}
if (!string.IsNullOrEmpty(url))
{
// Initialize and store the HTTP client.
var credentialManager = HostContext.GetService <ICredentialManager>();
// Get the auth type. On premise defaults to negotiate (Kerberos with fallback to NTLM).
// Hosted defaults to PAT authentication.
string defaultAuthType = UrlUtil.IsHosted(url) ? Constants.Configuration.PAT :
(Constants.Agent.Platform == Constants.OSPlatform.Windows ? Constants.Configuration.Integrated : Constants.Configuration.Negotiate);
string authType = command.GetAuth(defaultValue: defaultAuthType);
ICredentialProvider provider = credentialManager.GetCredentialProvider(authType);
provider.EnsureCredential(HostContext, command, url);
_taskStore.HttpClient = new TaskAgentHttpClient(new Uri(url), provider.GetVssCredentials(HostContext));
}
var configStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configStore.GetSettings();
// Create job message.
IJobDispatcher jobDispatcher = null;
try
{
jobDispatcher = HostContext.CreateService <IJobDispatcher>();
foreach (JobInfo job in await ConvertToJobMessagesAsync(process, repoDirectory, token))
{
job.RequestMessage.Environment.Variables[Constants.Variables.Agent.RunMode] = RunMode.Local.ToString();
jobDispatcher.Run(job.RequestMessage);
Task jobDispatch = jobDispatcher.WaitAsync(token);
if (!Task.WaitAll(new[] { jobDispatch }, job.Timeout))
{
jobDispatcher.Cancel(job.CancelMessage);
// Finish waiting on the same job dispatch task. The first call to WaitAsync dequeues
// the dispatch task and then proceeds to wait on it. So we need to continue awaiting
// the task instance (queue is now empty).
await jobDispatch;
}
}
}
finally
{
if (jobDispatcher != null)
{
await jobDispatcher.ShutdownAsync();
}
}
return(Constants.Agent.ReturnCode.Success);
}
public async Task <int> RunAsync(CommandSettings command, CancellationToken token)
{
Trace.Info(nameof(RunAsync));
var configStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configStore.GetSettings();
// Store the HTTP client.
// todo: fix in master to allow URL to be empty and then rebase on master.
const string DefaultUrl = "http://127.0.0.1/local-runner-default-url";
string url = command.GetUrl(DefaultUrl);
if (!string.Equals(url, DefaultUrl, StringComparison.Ordinal))
{
var credentialManager = HostContext.GetService <ICredentialManager>();
string authType = command.GetAuth(defaultValue: Constants.Configuration.Integrated);
ICredentialProvider provider = credentialManager.GetCredentialProvider(authType);
provider.EnsureCredential(HostContext, command, url);
_httpClient = new TaskAgentHttpClient(new Uri(url), provider.GetVssCredentials(HostContext));
}
// Load the YAML file.
string yamlFile = command.GetYaml();
ArgUtil.File(yamlFile, nameof(yamlFile));
var parseOptions = new ParseOptions
{
MaxFiles = 10,
MustacheEvaluationMaxResultLength = 512 * 1024, // 512k string length
MustacheEvaluationTimeout = TimeSpan.FromSeconds(10),
MustacheMaxDepth = 5,
};
var pipelineParser = new PipelineParser(new PipelineTraceWriter(), new PipelineFileProvider(), parseOptions);
Pipelines.Process process = pipelineParser.Load(
defaultRoot: Directory.GetCurrentDirectory(),
path: yamlFile,
mustacheContext: null,
cancellationToken: HostContext.AgentShutdownToken);
ArgUtil.NotNull(process, nameof(process));
if (command.WhatIf)
{
return(Constants.Agent.ReturnCode.Success);
}
// Create job message.
IJobDispatcher jobDispatcher = null;
try
{
jobDispatcher = HostContext.CreateService <IJobDispatcher>();
foreach (JobInfo job in await ConvertToJobMessagesAsync(process, token))
{
job.RequestMessage.Environment.Variables[Constants.Variables.Agent.RunMode] = RunMode.Local.ToString();
jobDispatcher.Run(job.RequestMessage);
Task jobDispatch = jobDispatcher.WaitAsync(token);
if (!Task.WaitAll(new[] { jobDispatch }, job.Timeout))
{
jobDispatcher.Cancel(job.CancelMessage);
// Finish waiting on the same job dispatch task. The first call to WaitAsync dequeues
// the dispatch task and then proceeds to wait on it. So we need to continue awaiting
// the task instance (queue is now empty).
await jobDispatch;
}
}
}
finally
{
if (jobDispatcher != null)
{
await jobDispatcher.ShutdownAsync();
}
}
return(Constants.Agent.ReturnCode.Success);
}
private async Task <List <JobInfo> > ConvertToJobMessagesAsync(Pipelines.Process process, string repoDirectory, CancellationToken token)
{
// Collect info about the repo.
string repoName = Path.GetFileName(repoDirectory);
string userName = await GitAsync("config --get user.name", token);
string userEmail = await GitAsync("config --get user.email", token);
string branch = await GitAsync("symbolic-ref HEAD", token);
string commit = await GitAsync("rev-parse HEAD", token);
string commitAuthorName = await GitAsync("show --format=%an --no-patch HEAD", token);
string commitSubject = await GitAsync("show --format=%s --no-patch HEAD", token);
var jobs = new List <JobInfo>();
int requestId = 1;
foreach (Phase phase in process.Phases ?? new List <IPhase>(0))
{
foreach (Job job in phase.Jobs ?? new List <IJob>(0))
{
var builder = new StringBuilder();
builder.Append($@"{{
""tasks"": [");
var steps = new List <ISimpleStep>();
foreach (IStep step in job.Steps ?? new List <IStep>(0))
{
if (step is ISimpleStep)
{
steps.Add(step as ISimpleStep);
}
else
{
var stepsPhase = step as StepsPhase;
foreach (ISimpleStep nestedStep in stepsPhase.Steps ?? new List <ISimpleStep>(0))
{
steps.Add(nestedStep);
}
}
}
bool firstStep = true;
foreach (ISimpleStep step in steps)
{
if (!(step is TaskStep))
{
throw new Exception("Unable to run step type: " + step.GetType().FullName);
}
var task = step as TaskStep;
if (!task.Enabled)
{
continue;
}
// Sanity check - the pipeline parser should have already validated version is an int.
int taskVersion;
if (!int.TryParse(task.Reference.Version, NumberStyles.None, CultureInfo.InvariantCulture, out taskVersion))
{
throw new Exception($"Unexpected task version format. Expected an unsigned integer with no formatting. Actual: '{taskVersion}'");
}
TaskDefinition definition = await _taskStore.GetTaskAsync(
name : task.Reference.Name,
version : taskVersion,
token : token);
await _taskStore.EnsureCachedAsync(definition, token);
if (!firstStep)
{
builder.Append(",");
}
firstStep = false;
builder.Append($@"
{{
""instanceId"": ""{Guid.NewGuid()}"",
""displayName"": {JsonConvert.ToString(!string.IsNullOrEmpty(task.Name) ? task.Name : definition.InstanceNameFormat)},
""enabled"": true,
""continueOnError"": {task.ContinueOnError.ToString().ToLowerInvariant()},
""condition"": {JsonConvert.ToString(task.Condition)},
""alwaysRun"": false,
""timeoutInMinutes"": {task.TimeoutInMinutes.ToString(CultureInfo.InvariantCulture)},
""id"": ""{definition.Id}"",
""name"": {JsonConvert.ToString(definition.Name)},
""version"": {JsonConvert.ToString(definition.Version.ToString())},
""inputs"": {{");
bool firstInput = true;
foreach (KeyValuePair <string, string> input in task.Inputs ?? new Dictionary <string, string>(0))
{
if (!firstInput)
{
builder.Append(",");
}
firstInput = false;
builder.Append($@"
{JsonConvert.ToString(input.Key)}: {JsonConvert.ToString(input.Value)}");
}
builder.Append($@"
}},
""environment"": {{");
bool firstEnv = true;
foreach (KeyValuePair <string, string> env in task.Environment ?? new Dictionary <string, string>(0))
{
if (!firstEnv)
{
builder.Append(",");
}
firstEnv = false;
builder.Append($@"
{JsonConvert.ToString(env.Key)}: {JsonConvert.ToString(env.Value)}");
}
builder.Append($@"
}}
}}");
}
builder.Append($@"
],
""requestId"": {requestId++},
""lockToken"": ""00000000-0000-0000-0000-000000000000"",
""lockedUntil"": ""0001-01-01T00:00:00"",
""messageType"": ""JobRequest"",
""plan"": {{
""scopeIdentifier"": ""00000000-0000-0000-0000-000000000000"",
""planType"": ""Build"",
""version"": 8,
""planId"": ""00000000-0000-0000-0000-000000000000"",
""artifactUri"": ""vstfs:///Build/Build/1234"",
""artifactLocation"": null
}},
""timeline"": {{
""id"": ""00000000-0000-0000-0000-000000000000"",
""changeId"": 1,
""location"": null
}},
""jobId"": ""{Guid.NewGuid()}"",
""jobName"": {JsonConvert.ToString(!string.IsNullOrEmpty(job.Name) ? job.Name : "Build")},
""environment"": {{
""endpoints"": [
{{
""data"": {{
""repositoryId"": ""00000000-0000-0000-0000-000000000000"",
""localDirectory"": {JsonConvert.ToString(repoDirectory)},
""clean"": ""false"",
""checkoutSubmodules"": ""False"",
""onpremtfsgit"": ""False"",
""fetchDepth"": ""0"",
""gitLfsSupport"": ""false"",
""skipSyncSource"": ""false"",
""cleanOptions"": ""0""
}},
""name"": {JsonConvert.ToString(repoName)},
""type"": ""LocalRun"",
""url"": ""https://127.0.0.1/vsts-agent-local-runner?directory={Uri.EscapeDataString(repoDirectory)}"",
""authorization"": {{
""parameters"": {{
""AccessToken"": ""dummy-access-token""
}},
""scheme"": ""OAuth""
}},
""isReady"": false
}}
],
""mask"": [
{{
""type"": ""regex"",
""value"": ""dummy-access-token""
}}
],
""variables"": {{");
builder.Append($@"
""system"": ""build"",
""system.collectionId"": ""00000000-0000-0000-0000-000000000000"",
""system.culture"": ""en-US"",
""system.definitionId"": ""55"",
""system.isScheduled"": ""False"",
""system.hosttype"": ""build"",
""system.jobId"": ""00000000-0000-0000-0000-000000000000"",
""system.planId"": ""00000000-0000-0000-0000-000000000000"",
""system.timelineId"": ""00000000-0000-0000-0000-000000000000"",
""system.taskDefinitionsUri"": ""https://127.0.0.1/vsts-agent-local-runner"",
""system.teamFoundationCollectionUri"": ""https://127.0.0.1/vsts-agent-local-runner"",
""system.teamProject"": {JsonConvert.ToString(repoName)},
""system.teamProjectId"": ""00000000-0000-0000-0000-000000000000"",
""build.buildId"": ""1863"",
""build.buildNumber"": ""1863"",
""build.buildUri"": ""vstfs:///Build/Build/1863"",
""build.clean"": """",
""build.definitionName"": ""My Build Definition Name"",
""build.definitionVersion"": ""1"",
""build.queuedBy"": {JsonConvert.ToString(userName)},
""build.queuedById"": ""00000000-0000-0000-0000-000000000000"",
""build.requestedFor"": {JsonConvert.ToString(userName)},
""build.requestedForEmail"": {JsonConvert.ToString(userEmail)},
""build.requestedForId"": ""00000000-0000-0000-0000-000000000000"",
""build.repository.uri"": ""https://127.0.0.1/vsts-agent-local-runner/_git/{Uri.EscapeDataString(repoName)}"",
""build.sourceBranch"": {JsonConvert.ToString(branch)},
""build.sourceBranchName"": {JsonConvert.ToString(branch.Split('/').Last())},
""build.sourceVersion"": {JsonConvert.ToString(commit)},
""build.sourceVersionAuthor"": {JsonConvert.ToString(commitAuthorName)},
""build.sourceVersionMessage"": {JsonConvert.ToString(commitSubject)},
""AZURE_HTTP_USER_AGENT"": ""VSTS_00000000-0000-0000-0000-000000000000_build_55_1863"",
""MSDEPLOY_HTTP_USER_AGENT"": ""VSTS_00000000-0000-0000-0000-000000000000_build_55_1863""");
foreach (Variable variable in job.Variables ?? new List <IVariable>(0))
{
builder.Append($@",
{JsonConvert.ToString(variable.Name ?? string.Empty)}: {JsonConvert.ToString(variable.Value ?? string.Empty)}");
}
builder.Append($@"
}},
""systemConnection"": {{
""data"": {{
""ServerId"": ""00000000-0000-0000-0000-000000000000"",
""ServerName"": ""127.0.0.1""
}},
""name"": ""SystemVssConnection"",
""url"": ""https://127.0.0.1/vsts-agent-local-runner"",
""authorization"": {{
""parameters"": {{
""AccessToken"": ""dummy-access-token""
}},
""scheme"": ""OAuth""
}},
""isReady"": false
}}
}}
}}");
string message = builder.ToString();
try
{
jobs.Add(new JobInfo(job, message));
}
catch
{
Dump("Job message JSON", message);
throw;
}
}
}
return(jobs);
}
private async Task <List <JobInfo> > ConvertToJobMessagesAsync(Pipelines.Process process, CancellationToken token)
{
var jobs = new List <JobInfo>();
int requestId = 1;
foreach (Phase phase in process.Phases ?? new List <IPhase>(0))
{
foreach (Job job in phase.Jobs ?? new List <IJob>(0))
{
var builder = new StringBuilder();
builder.Append($@"{{
""tasks"": [");
var steps = new List <ISimpleStep>();
foreach (IStep step in job.Steps ?? new List <IStep>(0))
{
if (step is ISimpleStep)
{
steps.Add(step as ISimpleStep);
}
else
{
var stepsPhase = step as StepsPhase;
foreach (ISimpleStep nestedStep in stepsPhase.Steps ?? new List <ISimpleStep>(0))
{
steps.Add(nestedStep);
}
}
}
bool firstStep = true;
foreach (ISimpleStep step in steps)
{
if (!(step is TaskStep))
{
throw new Exception("Unable to run step type: " + step.GetType().FullName);
}
var task = step as TaskStep;
if (!task.Enabled)
{
continue;
}
TaskDefinition definition = await GetDefinitionAsync(task, token);
if (!firstStep)
{
builder.Append(",");
}
firstStep = false;
builder.Append($@"
{{
""instanceId"": ""{Guid.NewGuid()}"",
""displayName"": {JsonConvert.ToString(!string.IsNullOrEmpty(task.Name) ? task.Name : definition.InstanceNameFormat)},
""enabled"": true,
""continueOnError"": {task.ContinueOnError.ToString().ToLowerInvariant()},
""condition"": {JsonConvert.ToString(task.Condition)},
""alwaysRun"": false,
""timeoutInMinutes"": {task.TimeoutInMinutes.ToString(CultureInfo.InvariantCulture)},
""id"": ""{definition.Id}"",
""name"": {JsonConvert.ToString(definition.Name)},
""version"": {JsonConvert.ToString(GetVersion(definition).ToString())},
""inputs"": {{");
bool firstInput = true;
foreach (KeyValuePair <string, string> input in task.Inputs ?? new Dictionary <string, string>(0))
{
if (!firstInput)
{
builder.Append(",");
}
firstInput = false;
builder.Append($@"
{JsonConvert.ToString(input.Key)}: {JsonConvert.ToString(input.Value)}");
}
builder.Append($@"
}},
""environment"": {{");
bool firstEnv = true;
foreach (KeyValuePair <string, string> env in task.Environment ?? new Dictionary <string, string>(0))
{
if (!firstEnv)
{
builder.Append(",");
}
firstEnv = false;
builder.Append($@"
{JsonConvert.ToString(env.Key)}: {JsonConvert.ToString(env.Value)}");
}
builder.Append($@"
}}
}}");
}
builder.Append($@"
],
""requestId"": {requestId++},
""lockToken"": ""00000000-0000-0000-0000-000000000000"",
""lockedUntil"": ""0001-01-01T00:00:00"",
""messageType"": ""JobRequest"",
""plan"": {{
""scopeIdentifier"": ""00000000-0000-0000-0000-000000000000"",
""planType"": ""Build"",
""version"": 8,
""planId"": ""00000000-0000-0000-0000-000000000000"",
""artifactUri"": ""vstfs:///Build/Build/1234"",
""artifactLocation"": null
}},
""timeline"": {{
""id"": ""00000000-0000-0000-0000-000000000000"",
""changeId"": 1,
""location"": null
}},
""jobId"": ""{Guid.NewGuid()}"",
""jobName"": {JsonConvert.ToString(!string.IsNullOrEmpty(job.Name) ? job.Name : "Build")},
""environment"": {{
""endpoints"": [
{{
""data"": {{
""repositoryId"": ""00000000-0000-0000-0000-000000000000"",
""rootFolder"": null,
""clean"": ""false"",
""checkoutSubmodules"": ""False"",
""onpremtfsgit"": ""False"",
""fetchDepth"": ""0"",
""gitLfsSupport"": ""false"",
""skipSyncSource"": ""true"",
""cleanOptions"": ""0""
}},
""name"": ""gitTest"",
""type"": ""TfsGit"",
""url"": ""https://127.0.0.1/vsts-agent-local-runner/_git/gitTest"",
""authorization"": {{
""parameters"": {{
""AccessToken"": ""dummy-access-token""
}},
""scheme"": ""OAuth""
}},
""isReady"": false
}}
],
""mask"": [
{{
""type"": ""regex"",
""value"": ""dummy-access-token""
}}
],
""variables"": {{");
builder.Append($@"
""system"": ""build"",
""system.collectionId"": ""00000000-0000-0000-0000-000000000000"",
""system.teamProject"": ""gitTest"",
""system.teamProjectId"": ""00000000-0000-0000-0000-000000000000"",
""system.definitionId"": ""55"",
""build.definitionName"": ""My Build Definition Name"",
""build.definitionVersion"": ""1"",
""build.queuedBy"": ""John Doe"",
""build.queuedById"": ""00000000-0000-0000-0000-000000000000"",
""build.requestedFor"": ""John Doe"",
""build.requestedForId"": ""00000000-0000-0000-0000-000000000000"",
""build.requestedForEmail"": ""*****@*****.**"",
""build.sourceVersion"": ""55ba1763b74d42a758514b466b7ea931aedbc941"",
""build.sourceBranch"": ""refs/heads/master"",
""build.sourceBranchName"": ""master"",
""system.culture"": ""en-US"",
""build.clean"": """",
""build.buildId"": ""1863"",
""build.buildUri"": ""vstfs:///Build/Build/1863"",
""build.buildNumber"": ""1863"",
""system.isScheduled"": ""False"",
""system.hosttype"": ""build"",
""system.teamFoundationCollectionUri"": ""https://127.0.0.1/vsts-agent-local-runner"",
""system.taskDefinitionsUri"": ""https://127.0.0.1/vsts-agent-local-runner"",
""AZURE_HTTP_USER_AGENT"": ""VSTS_00000000-0000-0000-0000-000000000000_build_55_1863"",
""MSDEPLOY_HTTP_USER_AGENT"": ""VSTS_00000000-0000-0000-0000-000000000000_build_55_1863"",
""system.planId"": ""00000000-0000-0000-0000-000000000000"",
""system.jobId"": ""00000000-0000-0000-0000-000000000000"",
""system.timelineId"": ""00000000-0000-0000-0000-000000000000"",
""build.repository.uri"": ""https://127.0.0.1/vsts-agent-local-runner/_git/gitTest"",
""build.sourceVersionAuthor"": ""John Doe"",
""build.sourceVersionMessage"": ""Updated Program.cs""");
foreach (Variable variable in job.Variables ?? new List <IVariable>(0))
{
builder.Append($@",
{JsonConvert.ToString(variable.Name ?? string.Empty)}: {JsonConvert.ToString(variable.Value ?? string.Empty)}");
}
builder.Append($@"
}},
""systemConnection"": {{
""data"": {{
""ServerId"": ""00000000-0000-0000-0000-000000000000"",
""ServerName"": ""127.0.0.1""
}},
""name"": ""SystemVssConnection"",
""url"": ""https://127.0.0.1/vsts-agent-local-runner"",
""authorization"": {{
""parameters"": {{
""AccessToken"": ""dummy-access-token""
}},
""scheme"": ""OAuth""
}},
""isReady"": false
}}
}}
}}");
string message = builder.ToString();
try
{
jobs.Add(new JobInfo(job, message));
}
catch
{
Dump("Job message JSON", message);
throw;
}
}
}
return(jobs);
}