private void InitializeSecretMasker(JobRequestMessage message) { Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); var secretMasker = HostContext.GetService <ISecretMasker>(); // Add mask hints var variables = message?.Environment?.Variables ?? new Dictionary <string, string>(); foreach (MaskHint maskHint in (message.Environment.MaskHints ?? new List <MaskHint>())) { if (maskHint.Type == MaskType.Regex) { secretMasker.AddRegex(maskHint.Value); // Also add the JSON escaped string since the job message is traced in the diag log. secretMasker.AddValue(JsonConvert.ToString(maskHint.Value ?? string.Empty)); } else if (maskHint.Type == MaskType.Variable) { string value; if (variables.TryGetValue(maskHint.Value, out value) && !string.IsNullOrEmpty(value)) { secretMasker.AddValue(value); // Also add the JSON escaped string since the job message is traced in the diag log. secretMasker.AddValue(JsonConvert.ToString(value)); } } else { // TODO: Should we fail instead? Do any additional pains need to be taken here? Should the job message not be traced? Trace.Warning($"Unsupported mask type '{maskHint.Type}'."); } } // TODO: Avoid adding redundant secrets. If the endpoint auth matches the system connection, then it's added as a value secret and as a regex secret. Once as a value secret b/c of the following code that iterates over each endpoint. Once as a regex secret due to the hint sent down in the job message. // Add masks for service endpoints foreach (ServiceEndpoint endpoint in message.Environment.Endpoints ?? new List <ServiceEndpoint>()) { foreach (string value in endpoint.Authorization?.Parameters?.Values ?? new string[0]) { if (!string.IsNullOrEmpty(value)) { secretMasker.AddValue(value); // This is precautionary if the secret is used in an URL. For example, if "allow scripts // access to OAuth token" is checked, then the repository auth key is injected into the // URL for a Git repository's remote configuration. if (!Uri.EscapeDataString(value).Equals(value, StringComparison.OrdinalIgnoreCase)) { secretMasker.AddValue(Uri.EscapeDataString(value)); } } } } }
public void InitializeJob(JobRequestMessage message, CancellationToken token) { // Validate/store parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection)); ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); CancellationToken = token; // Initialize the environment. Endpoints = message.Environment.Endpoints; // add system connect to the endpoint list. Endpoints.Add(message.Environment.SystemConnection); List <string> warnings; Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings); // Initialize the job timeline record. // the job timeline record is at order 1. InitializeTimelineRecord(message.Timeline.Id, message.JobId, null, ExecutionContextType.Job, message.JobName, 1); // Log any warnings. foreach (string warning in (warnings ?? new List <string>())) { this.Warning(warning); } // Set system.debug WriteDebug = Variables.System_Debug ?? false; }
private void InitializeSecretMasker(JobRequestMessage message) { Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); var secretMasker = HostContext.GetService<ISecretMasker>(); // Add mask hints var variables = message?.Environment?.Variables ?? new Dictionary<string, string>(); foreach (MaskHint maskHint in (message.Environment.MaskHints ?? new List<MaskHint>())) { if (maskHint.Type == MaskType.Regex) { secretMasker.AddRegex(maskHint.Value); // Also add the JSON escaped string since the job message is traced in the diag log. secretMasker.AddValue(JsonConvert.ToString(maskHint.Value ?? string.Empty)); } else if (maskHint.Type == MaskType.Variable) { string value; if (variables.TryGetValue(maskHint.Value, out value) && !string.IsNullOrEmpty(value)) { secretMasker.AddValue(value); // Also add the JSON escaped string since the job message is traced in the diag log. secretMasker.AddValue(JsonConvert.ToString(value)); } } else { // TODO: Should we fail instead? Do any additional pains need to be taken here? Should the job message not be traced? Trace.Warning($"Unsupported mask type '{maskHint.Type}'."); } } // TODO: Avoid adding redundant secrets. If the endpoint auth matches the system connection, then it's added as a value secret and as a regex secret. Once as a value secret b/c of the following code that iterates over each endpoint. Once as a regex secret due to the hint sent down in the job message. // Add masks for service endpoints foreach (ServiceEndpoint endpoint in message.Environment.Endpoints ?? new List<ServiceEndpoint>()) { foreach (string value in endpoint.Authorization?.Parameters?.Values ?? new string[0]) { secretMasker.AddValue(value); // This is precautionary if the secret is used in an URL. For example, if "allow scripts // access to OAuth token" is checked, then the repository auth key is injected into the // URL for a Git repository's remote configuration. if (!Uri.EscapeDataString(value).Equals(value, StringComparison.OrdinalIgnoreCase)) { secretMasker.AddValue(Uri.EscapeDataString(value)); } } } }
private JobRequestMessage CreateJobRequestMessage() { TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; JobEnvironment environment = new JobEnvironment(); List<TaskInstance> tasks = new List<TaskInstance>(); Guid JobId = Guid.NewGuid(); var jobRequest = new JobRequestMessage(plan, timeline, JobId, "someJob", environment, tasks); return jobRequest; }
private JobRequestMessage CreateJobRequestMessage(string jobName) { TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; JobEnvironment environment = new JobEnvironment(); List <TaskInstance> tasks = new List <TaskInstance>(); Guid JobId = Guid.NewGuid(); var jobRequest = new JobRequestMessage(plan, timeline, JobId, jobName, environment, tasks); return(jobRequest); }
public override async Task <int> HandleJobRequest(JobRequestMessage message) { var submission = await Context.Submissions.FindAsync(message.TargetId); if (submission is null || submission.RequestVersion >= message.RequestVersion) { Logger.LogDebug($"IgnoreJudgeRequestMessage" + $" SubmissionId={message.TargetId}" + $" RequestVersion={message.RequestVersion}"); return(0); }
public override async Task <int> HandleJobRequest(JobRequestMessage message) { var plagiarism = await Context.Plagiarisms.FindAsync(message.TargetId); if (plagiarism is null || plagiarism.RequestVersion >= message.RequestVersion) { Logger.LogDebug($"IgnoreCheckPlagiarismMessage" + $" PlagiarismId={message.TargetId}" + $" RequestVersion={message.RequestVersion}"); return(0); }
private JobRequestMessage CreateJobRequestMessage(string jobName) { TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; JobEnvironment environment = new JobEnvironment(); environment.Variables[Constants.Variables.System.Culture] = "en-US"; List<TaskInstance> tasks = new List<TaskInstance>(); Guid JobId = Guid.NewGuid(); var jobRequest = new JobRequestMessage(plan, timeline, JobId, jobName, environment, tasks); return jobRequest; }
public void TestRun() { using (var hc = new TestHostContext(this)) using (var workerManager = new WorkerManager()) { hc.EnqueueInstance <IJobDispatcher>(_jobDispatcher.Object); workerManager.Initialize(hc); JobRequestMessage jobMessage = CreateJobRequestMessage(); _jobDispatcher.Setup(x => x.RunAsync(jobMessage, It.IsAny <CancellationToken>())) .Returns(Task.FromResult <int>(21)); workerManager.Run(jobMessage); _jobDispatcher.Verify(x => x.RunAsync(jobMessage, It.IsAny <CancellationToken>()), "IJobDispatcher.RunAsync not invoked"); } }
private void SetCulture(JobRequestMessage message) { // Extract the culture name from the job's variable dictionary. // The variable does not exist for TFS 2015 RTM and Update 1. // It was introduced in Update 2. string culture; ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); if (message.Environment.Variables.TryGetValue(Constants.Variables.System.Culture, out culture)) { // Set the default thread culture. HostContext.SetDefaultCulture(culture); } }
public void Run(JobRequestMessage jobRequestMessage) { Trace.Info("Job request {0} received.", jobRequestMessage.JobId); var jobDispatcher = HostContext.CreateService <IJobDispatcher>(); Task <int> jobDispatcherTask; var cancellationTokenSource = new CancellationTokenSource(); jobDispatcherTask = jobDispatcher.RunAsync(jobRequestMessage, cancellationTokenSource.Token); var postJobDispatcherAction = new Action <Task <int>, Object>((task, obj) => { lock (_lock) { //protect from running after WorkerManager has been disposed if ((obj as WorkerManager)._disposed) { return; } if (task.Status == TaskStatus.Canceled) { Trace.Info("Job request {0} was canceled.", jobRequestMessage.JobId); } else if (task.Status == TaskStatus.Faulted) { Trace.Error("Job request {0} failed witn an exception.", jobRequestMessage.JobId); Trace.Error(task.Exception); } else { Trace.Info("Job request {0} processed with return code {1}.", jobRequestMessage.JobId, task.Result); } JobDispatcherItem deletedJob; if (_jobsInProgress.TryGetValue(jobRequestMessage.JobId, out deletedJob)) { _jobsInProgress.Remove(jobRequestMessage.JobId); deletedJob.Dispose(); } } }); lock (_lock) { var jobItem = new JobDispatcherItem(jobDispatcher, null, cancellationTokenSource); // store the new job item, so that it can be cancelled later if needed _jobsInProgress[jobRequestMessage.JobId] = jobItem; //run code after the job completes, which prints the result int the trace jobItem.DispatcherTask = jobDispatcherTask.ContinueWith(postJobDispatcherAction, this); } }
public async void DispatchesJobRequest() { //Arrange using (var hc = new TestHostContext(this)) { var jobDispatcher = new JobDispatcher(); hc.SetSingleton <IConfigurationStore>(_configurationStore.Object); hc.SetSingleton <IAgentServer>(_agentServer.Object); hc.EnqueueInstance <IProcessChannel>(_processChannel.Object); hc.EnqueueInstance <IProcessInvoker>(_processInvoker.Object); _configurationStore.Setup(x => x.GetSettings()).Returns(new AgentSettings() { PoolId = 1 }); jobDispatcher.Initialize(hc); var ts = new CancellationTokenSource(); JobRequestMessage message = CreateJobRequestMessage(); string strMessage = JsonUtility.ToString(message); _processInvoker.Setup(x => x.ExecuteAsync(It.IsAny <String>(), It.IsAny <String>(), "spawnclient 1 2", null, It.IsAny <CancellationToken>())) .Returns(Task.FromResult <int>(56)); _processChannel.Setup(x => x.StartServer(It.IsAny <StartProcessDelegate>())) .Callback((StartProcessDelegate startDel) => { startDel("1", "2"); }); _processChannel.Setup(x => x.SendAsync(MessageType.NewJobRequest, It.Is <string>(s => s.Equals(strMessage)), It.IsAny <CancellationToken>())) .Returns(Task.CompletedTask); var request = new TaskAgentJobRequest(); PropertyInfo sessionIdProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); Assert.NotNull(sessionIdProperty); sessionIdProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); _agentServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny <int>(), It.IsAny <long>(), It.IsAny <Guid>(), It.IsAny <CancellationToken>())).Returns(Task.FromResult <TaskAgentJobRequest>(request)); _agentServer.Setup(x => x.FinishAgentRequestAsync(It.IsAny <int>(), It.IsAny <long>(), It.IsAny <Guid>(), It.IsAny <DateTime>(), It.IsAny <TaskResult>(), It.IsAny <CancellationToken>())).Returns(Task.FromResult <TaskAgentJobRequest>(new TaskAgentJobRequest())); //Actt jobDispatcher.Run(message); //Assert await jobDispatcher.WaitAsync(CancellationToken.None); } }
private void SetCulture(JobRequestMessage message) { // Extract the culture name from the job's variable dictionary. string culture; ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); if (!message.Environment.Variables.TryGetValue(Constants.Variables.System.Culture, out culture)) { culture = null; } // Set the default thread culture. // TODO: Does this validation need to be removed? The variable does not exist prior to 2015 Update 2. ArgUtil.NotNullOrEmpty(culture, nameof(culture)); HostContext.SetDefaultCulture(culture); }
public void InitializeJob(JobRequestMessage message, CancellationToken token) { // Validate/store parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection)); ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); // Initialize the environment. Endpoints = message.Environment.Endpoints; // Add the system connection to the endpoint list. Endpoints.Add(message.Environment.SystemConnection); // Initialize the variables. The constructor handles the initial recursive expansion. List <string> warnings; Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings); // Initialize the job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, name: message.JobName, order: 1); // The job timeline record must be at order 1. // Initialize the logger before writing warnings. _logger = HostContext.CreateService <IPagingLogger>(); _logger.Setup(_mainTimelineId, _record.Id); // Log any warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Initialize the verbosity (based on system.debug). WriteDebug = Variables.System_Debug ?? false; // Hook up JobServerQueueThrottling event, we will log warning on server tarpit. _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; }
public async void TestCancel() { //Arrange using (var hc = new TestHostContext(this)) using (var workerManager = new WorkerManager()) { hc.EnqueueInstance <IJobDispatcher>(_jobDispatcher.Object); workerManager.Initialize(hc); JobRequestMessage jobMessage = CreateJobRequestMessage(); JobCancelMessage cancelMessage = CreateJobCancelMessage(jobMessage.JobId); bool started = false; Task jobTask = null; _jobDispatcher.Setup(x => x.RunAsync(jobMessage, It.IsAny <CancellationToken>())) .Returns(async(JobRequestMessage message, CancellationToken token) => { jobTask = Task.Delay(5000, token); started = true; await jobTask; return(0); }); workerManager.Run(jobMessage); int i = 20; while (i > 0 && (!started)) { await Task.Delay(10); i--; } Assert.True(started); //Act //send cancel message workerManager.Cancel(cancelMessage); //Assert //wait up to 2 sec for cancellation to be processed Task[] taskToWait = { jobTask, Task.Delay(2000) }; await Task.WhenAny(taskToWait); _jobDispatcher.Verify(x => x.RunAsync(jobMessage, It.IsAny <CancellationToken>()), $"{nameof(_jobDispatcher.Object.RunAsync)} not invoked"); Assert.True(jobTask.IsCompleted); Assert.True(jobTask.IsCanceled); } }
// The server only send down OAuth token in Job Request message. public static VssConnection GetVssConnection(JobRequestMessage jobRequest) { ArgUtil.NotNull(jobRequest, nameof(jobRequest)); ArgUtil.NotNull(jobRequest.Environment, nameof(jobRequest.Environment)); ArgUtil.NotNull(jobRequest.Environment.SystemConnection, nameof(jobRequest.Environment.SystemConnection)); ArgUtil.NotNull(jobRequest.Environment.SystemConnection.Url, nameof(jobRequest.Environment.SystemConnection.Url)); Uri serverUrl = jobRequest.Environment.SystemConnection.Url; var credentials = GetVssCredential(jobRequest.Environment.SystemConnection); if (credentials == null) { throw new ArgumentNullException(nameof(credentials)); } else { return(CreateConnection(serverUrl, credentials)); } }
// The server only send down OAuth token in Job Request message. public static VssConnection GetVssConnection(JobRequestMessage jobRequest) { ArgUtil.NotNull(jobRequest, nameof(jobRequest)); ArgUtil.NotNull(jobRequest.Environment, nameof(jobRequest.Environment)); ArgUtil.NotNull(jobRequest.Environment.SystemConnection, nameof(jobRequest.Environment.SystemConnection)); ArgUtil.NotNull(jobRequest.Environment.SystemConnection.Url, nameof(jobRequest.Environment.SystemConnection.Url)); Uri serverUrl = jobRequest.Environment.SystemConnection.Url; var credentials = GetVssCredential(jobRequest.Environment.SystemConnection); if (credentials == null) { throw new ArgumentNullException(nameof(credentials)); } else { return CreateConnection(serverUrl, credentials); } }
public void Start(JobRequestMessage jobRequest) { Trace.Entering(); if (HostContext.RunMode == RunMode.Local) { _term = HostContext.GetService <ITerminal>(); return; } if (_queueInProcess) { Trace.Info("No-opt, all queue process tasks are running."); return; } ArgUtil.NotNull(jobRequest, nameof(jobRequest)); ArgUtil.NotNull(jobRequest.Plan, nameof(jobRequest.Plan)); ArgUtil.NotNull(jobRequest.Timeline, nameof(jobRequest.Timeline)); _scopeIdentifier = jobRequest.Plan.ScopeIdentifier; _hubName = jobRequest.Plan.PlanType; _planId = jobRequest.Plan.PlanId; _jobTimelineId = jobRequest.Timeline.Id; _jobTimelineRecordId = jobRequest.JobId; // Server already create the job timeline _timelineUpdateQueue[_jobTimelineId] = new ConcurrentQueue <TimelineRecord>(); _allTimelines.Add(_jobTimelineId); // Start three dequeue task Trace.Info("Start process web console line queue."); _webConsoleLineDequeueTask = ProcessWebConsoleLinesQueueAsync(); Trace.Info("Start process file upload queue."); _fileUploadDequeueTask = ProcessFilesUploadQueueAsync(); Trace.Info("Start process timeline update queue."); _timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync(); _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask }; _queueInProcess = true; }
public void Run(JobRequestMessage jobRequestMessage) { Trace.Info($"Job request {jobRequestMessage.JobId} received."); WorkerDispatcher currentDispatch = null; if (_jobDispatchedQueue.Count > 0) { Guid dispatchedJobId = _jobDispatchedQueue.Dequeue(); if (_jobInfos.TryGetValue(dispatchedJobId, out currentDispatch)) { Trace.Verbose($"Retrive previous WorkerDispather for job {currentDispatch.JobId}."); } } WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId); newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token); _jobInfos.TryAdd(newDispatch.JobId, newDispatch); _jobDispatchedQueue.Enqueue(newDispatch.JobId); }
public void InitializeJob(JobRequestMessage message, CancellationToken token) { // Validate/store parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection)); ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); CancellationToken = token; // Initialize the environment. Endpoints = message.Environment.Endpoints; // Add the system connection to the endpoint list. Endpoints.Add(message.Environment.SystemConnection); // Initialize the variables. The constructor handles the initial recursive expansion. List <string> warnings; Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings); // Initialize the job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, name: message.JobName, order: 1); // The job timeline record must be at order 1. // Log any warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Initialize the verbosity (based on system.debug). WriteDebug = Variables.System_Debug ?? false; }
private async Task CompleteJobRequestAsync(int poolId, JobRequestMessage message, Guid lockToken, TaskResult result) { var agentServer = HostContext.GetService <IAgentServer>(); try { using (var csFinishRequest = new CancellationTokenSource(ChannelTimeout)) { await agentServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, csFinishRequest.Token); } } catch (TaskAgentJobNotFoundException) { Trace.Info("TaskAgentJobNotFoundException received, job is no longer valid."); } catch (TaskAgentJobTokenExpiredException) { Trace.Info("TaskAgentJobTokenExpiredException received, job is no longer valid."); } // This should be the last thing to run so we don't notify external parties until actually finished var notification = HostContext.GetService <IJobNotification>(); await notification.JobCompleted(message.JobId); }
public void Start(JobRequestMessage jobRequest) { Trace.Entering(); if (_queueInProcess) { Trace.Info("No-opt, all queue process tasks are running."); return; } ArgUtil.NotNull(jobRequest, nameof(jobRequest)); ArgUtil.NotNull(jobRequest.Plan, nameof(jobRequest.Plan)); ArgUtil.NotNull(jobRequest.Timeline, nameof(jobRequest.Timeline)); _scopeIdentifier = jobRequest.Plan.ScopeIdentifier; _hubName = jobRequest.Plan.PlanType; _planId = jobRequest.Plan.PlanId; _jobTimelineId = jobRequest.Timeline.Id; _jobTimelineRecordId = jobRequest.JobId; // Server already create the job timeline _timelineUpdateQueue[_jobTimelineId] = new ConcurrentQueue<TimelineRecord>(); _allTimelines.Add(_jobTimelineId); // Start three dequeue task Trace.Info("Start process web console line queue."); _webConsoleLineDequeueTask = ProcessWebConsoleLinesQueueAsync(); Trace.Info("Start process file upload queue."); _fileUploadDequeueTask = ProcessFilesUploadQueueAsync(); Trace.Info("Start process timeline update queue."); _timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync(); _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask }; _queueInProcess = true; }
public async Task <bool> SendAsync(JobRequestMessage message) { var serialized = JsonConvert.SerializeObject(message); var body = Encoding.UTF8.GetBytes(serialized); using var scope = _factory.CreateScope(); var queueStatisticsService = scope.ServiceProvider.GetRequiredService <QueueStatisticsService>(); try { Channel.BasicPublish("", Queue, null, body); await queueStatisticsService.AddJobRequestAsync(message); Logger.LogDebug($"SendJobRequestMessage JobType={message.JobType}" + $" TargetId={message.TargetId} RequestVersion={message.RequestVersion}"); return(true); } catch (Exception e) { Logger.LogError($"SendJobRequestMessage failed: {e.Message}"); Logger.LogError($"Stacktrace: {e.StackTrace}"); return(false); } }
private void _read() { _protcolId = m_io.ReadU1(); _messageType = ((S7MessageType)m_io.ReadU1()); _reserved = m_io.ReadU2be(); _pduReference = m_io.ReadU2be(); _parametersLength = m_io.ReadU2be(); _dataLength = m_io.ReadU2be(); if (MessageType == S7MessageType.AckData) { _error = new ResponseError(m_io, this, m_root); } switch (MessageType) { case S7MessageType.JobRequest: { __raw_message = m_io.ReadBytes((ParametersLength + DataLength)); var io___raw_message = new KaitaiStream(__raw_message); _message = new JobRequestMessage(io___raw_message, this, m_root); break; } case S7MessageType.AckData: { __raw_message = m_io.ReadBytes((ParametersLength + DataLength)); var io___raw_message = new KaitaiStream(__raw_message); _message = new AckDataMessage(io___raw_message, this, m_root); break; } default: { __raw_message = m_io.ReadBytes((ParametersLength + DataLength)); var io___raw_message = new KaitaiStream(__raw_message); _message = new MessageOther(io___raw_message, this, m_root); break; } } }
public async Task <TaskResult> RunAsync(JobRequestMessage message, CancellationToken jobRequestCancellationToken) { // Validate parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); ArgUtil.NotNull(message.Tasks, nameof(message.Tasks)); Trace.Info("Job ID {0}", message.JobId); if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) && StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken])) { // TODO: get access token use Util Method message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"]; } // Setup the job server and job server queue. var jobServer = HostContext.GetService <IJobServer>(); var jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection); var jobConnection = ApiUtil.CreateConnection(ReplaceWithConfigUriBase(message.Environment.SystemConnection.Url), jobServerCredential); await jobServer.ConnectAsync(jobConnection); var jobServerQueue = HostContext.GetService <IJobServerQueue>(); jobServerQueue.Start(message); IExecutionContext jobContext = null; try { // Create the job execution context. jobContext = HostContext.CreateService <IExecutionContext>(); jobContext.InitializeJob(message, jobRequestCancellationToken); Trace.Info("Starting the job execution context."); jobContext.Start(); // Set agent variables. AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings(); jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture)); jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath()); jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName); jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName); jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName); jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext)); jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), "vstsom")); jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext)); jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext)); // prefer task definitions url, then TFS url var taskServer = HostContext.GetService <ITaskServer>(); string taskUrl = jobContext.Variables.System_TaskDefinitionsUri; Uri taskServerUri; if (string.IsNullOrEmpty(taskUrl)) { taskServerUri = ReplaceWithConfigUriBase(message.Environment.SystemConnection.Url); Trace.Info($"Creating task server with tfs server url {taskServerUri.ToString()}"); } else { taskServerUri = ReplaceWithConfigUriBase(new Uri(taskUrl)); Trace.Info($"Creating task server with {taskServerUri.ToString()}"); } var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection); await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential)); // Expand the endpoint data values. foreach (ServiceEndpoint endpoint in jobContext.Endpoints) { jobContext.Variables.ExpandValues(target: endpoint.Data); VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data); } // Get the job extensions. Trace.Info("Getting job extensions."); string hostType = jobContext.Variables.System_HostType; var extensionManager = HostContext.GetService <IExtensionManager>(); IJobExtension[] extensions = (extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>()) .Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase)) .ToArray(); // Add the prepare steps. Trace.Info("Adding job prepare extensions."); List <IStep> steps = new List <IStep>(); foreach (IJobExtension extension in extensions) { if (extension.PrepareStep != null) { Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}."); extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName); steps.Add(extension.PrepareStep); } } // Add the task steps. Trace.Info("Adding tasks."); foreach (TaskInstance taskInstance in message.Tasks) { Trace.Verbose($"Adding {taskInstance.DisplayName}."); var taskRunner = HostContext.CreateService <ITaskRunner>(); taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName); taskRunner.TaskInstance = taskInstance; steps.Add(taskRunner); } // Add the finally steps. Trace.Info("Adding job finally extensions."); foreach (IJobExtension extension in extensions) { if (extension.FinallyStep != null) { Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}."); extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName); steps.Add(extension.FinallyStep); } } // Download tasks if not already in the cache Trace.Info("Downloading task definitions."); var taskManager = HostContext.GetService <ITaskManager>(); try { await taskManager.DownloadAsync(jobContext, message.Tasks); } catch (OperationCanceledException ex) { // set the job to canceled Trace.Error($"Caught exception: {ex}"); jobContext.Error(ex); return(jobContext.Complete(TaskResult.Canceled)); } catch (Exception ex) { // Log the error and fail the job. Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}"); jobContext.Error(ex); return(jobContext.Complete(TaskResult.Failed)); } // Run the steps. var stepsRunner = HostContext.GetService <IStepsRunner>(); try { await stepsRunner.RunAsync(jobContext, steps); } catch (OperationCanceledException ex) { // set the job to canceled Trace.Error($"Caught exception: {ex}"); jobContext.Error(ex); return(jobContext.Complete(TaskResult.Canceled)); } catch (Exception ex) { // Log the error and fail the job. Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}"); jobContext.Error(ex); return(jobContext.Complete(TaskResult.Failed)); } Trace.Info($"Job result: {jobContext.Result}"); // Complete the job. Trace.Info("Completing the job execution context."); return(jobContext.Complete()); } finally { // Drain the job server queue. if (jobServerQueue != null) { try { Trace.Info("Shutting down the job server queue."); await jobServerQueue.ShutdownAsync(); } catch (Exception ex) { Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}"); } } } }
private async Task RunAsync(JobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken) { if (previousJobDispatch != null) { Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker."); await EnsureDispatchFinished(previousJobDispatch); } else { Trace.Verbose($"This is the first job request."); } var term = HostContext.GetService <ITerminal>(); term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName)); // first job request renew succeed. TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>(); // lock renew cancellation token. using (var lockRenewalTokenSource = new CancellationTokenSource()) using (var workerProcessCancelTokenSource = new CancellationTokenSource()) { long requestId = message.RequestId; Guid lockToken = message.LockToken; // start renew job request Trace.Info("Start renew job request."); Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token); // wait till first renew succeed or job request is canceled // not even start worker if the first renew fail await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken)); if (renewJobRequest.IsCompleted) { // renew job request task complete means we run out of retry for the first job request renew. // TODO: not need to return anything. Trace.Info("Unable to renew job request for the first time, stop dispatching job to worker."); return; } if (jobRequestCancellationToken.IsCancellationRequested) { await CompleteJobRequestAsync(_poolId, requestId, lockToken, TaskResult.Canceled); return; } Task <int> workerProcessTask = null; using (var processChannel = HostContext.CreateService <IProcessChannel>()) using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { // Start the process channel. // It's OK if StartServer bubbles an execption after the worker process has already started. // The worker will shutdown after 30 seconds if it hasn't received the job message. processChannel.StartServer( // Delegate to start the child process. startProcess: (string pipeHandleOut, string pipeHandleIn) => { // Validate args. ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut)); ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn)); // Start the child process. var assemblyDirectory = IOUtil.GetBinPath(); string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName); workerProcessTask = processInvoker.ExecuteAsync( workingDirectory: assemblyDirectory, fileName: workerFileName, arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn, environment: null, cancellationToken: workerProcessCancelTokenSource.Token); }); // Send the job request message. // Kill the worker process if sending the job message times out. The worker // process may have successfully received the job message. try { Trace.Info("Send job request message to worker."); using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout)) { await processChannel.SendAsync( messageType : MessageType.NewJobRequest, body : JsonUtility.ToString(message), cancellationToken : csSendJobRequest.Token); } } catch (OperationCanceledException) { // message send been cancelled. // timeout 45 sec. kill worker. Trace.Info("Job request message sending been cancelled, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { // worker process been killed. } Trace.Info("Stop renew job request."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; // not finish the job request since the job haven't run on worker at all, we will not going to set a result to server. return; } TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded; // wait for renewlock, worker process or cancellation token been fired. var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken)); if (completedTask == workerProcessTask) { // worker finished successfully, complete job request with result, stop renew lock, job has finished. int returnCode = await workerProcessTask; Trace.Info("Worker finished. Code: " + returnCode); TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode); Trace.Info($"finish job request with result: {result}"); term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result)); // complete job request await CompleteJobRequestAsync(_poolId, requestId, lockToken, result); Trace.Info("Stop renew job request."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; return; } else if (completedTask == renewJobRequest) { resultOnAbandonOrCancel = TaskResult.Abandoned; } else { resultOnAbandonOrCancel = TaskResult.Canceled; } // renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage) // cancel worker gracefully first, then kill it after 45 sec try { Trace.Info("Send job cancellation message to worker."); using (var csSendCancel = new CancellationTokenSource(ChannelTimeout)) { await processChannel.SendAsync( messageType : MessageType.CancelRequest, body : string.Empty, cancellationToken : csSendCancel.Token); } } catch (OperationCanceledException) { // message send been cancelled. Trace.Info("Job cancel message sending been cancelled, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { // worker process been killed. } } // wait worker to exit within 45 sec, then kill worker. using (var csKillWorker = new CancellationTokenSource(TimeSpan.FromSeconds(45))) { completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, csKillWorker.Token)); } // worker haven't exit within 45 sec. if (completedTask != workerProcessTask) { Trace.Info("worker process haven't exit after 45 sec, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { // worker process been killed. } } Trace.Info($"finish job request with result: {resultOnAbandonOrCancel}"); term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel)); // complete job request with cancel result, stop renew lock, job has finished. //TODO: don't finish job request on abandon await CompleteJobRequestAsync(_poolId, requestId, lockToken, resultOnAbandonOrCancel); Trace.Info("Stop renew job request."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; } } }
public void InitializeJob(JobRequestMessage message, CancellationToken token) { // Validate/store parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection)); ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); CancellationToken = token; // Initialize the environment. Endpoints = message.Environment.Endpoints; // Add the system connection to the endpoint list. Endpoints.Add(message.Environment.SystemConnection); // Initialize the variables. The constructor handles the initial recursive expansion. List<string> warnings; Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings); // Initialize the job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, name: message.JobName, order: 1); // The job timeline record must be at order 1. // Log any warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Initialize the verbosity (based on system.debug). WriteDebug = Variables.System_Debug ?? false; }
private async Task CompleteJobRequestAsync(int poolId, JobRequestMessage message, Guid lockToken, TaskResult result) { var agentServer = HostContext.GetService<IAgentServer>(); try { using (var csFinishRequest = new CancellationTokenSource(ChannelTimeout)) { await agentServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, csFinishRequest.Token); } } catch (TaskAgentJobNotFoundException) { Trace.Info("TaskAgentJobNotFoundException received, job is no longer valid."); } catch (TaskAgentJobTokenExpiredException) { Trace.Info("TaskAgentJobTokenExpiredException received, job is no longer valid."); } // This should be the last thing to run so we don't notify external parties until actually finished var notification = HostContext.GetService<IJobNotification>(); await notification.JobCompleted(message.JobId); }
private async Task RunAsync(JobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken) { if (previousJobDispatch != null) { Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker."); await EnsureDispatchFinished(previousJobDispatch); } else { Trace.Verbose($"This is the first job request."); } var term = HostContext.GetService<ITerminal>(); term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName)); // first job request renew succeed. TaskCompletionSource<int> firstJobRequestRenewed = new TaskCompletionSource<int>(); var notification = HostContext.GetService<IJobNotification>(); // lock renew cancellation token. using (var lockRenewalTokenSource = new CancellationTokenSource()) using (var workerProcessCancelTokenSource = new CancellationTokenSource()) { await notification.JobStarted(message.JobId); long requestId = message.RequestId; Guid lockToken = message.LockToken; // start renew job request Trace.Info("Start renew job request."); Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token); // wait till first renew succeed or job request is canceled // not even start worker if the first renew fail await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken)); if (renewJobRequest.IsCompleted) { // renew job request task complete means we run out of retry for the first job request renew. // TODO: not need to return anything. Trace.Info("Unable to renew job request for the first time, stop dispatching job to worker."); return; } if (jobRequestCancellationToken.IsCancellationRequested) { await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled); return; } Task<int> workerProcessTask = null; using (var processChannel = HostContext.CreateService<IProcessChannel>()) using (var processInvoker = HostContext.CreateService<IProcessInvoker>()) { // Start the process channel. // It's OK if StartServer bubbles an execption after the worker process has already started. // The worker will shutdown after 30 seconds if it hasn't received the job message. processChannel.StartServer( // Delegate to start the child process. startProcess: (string pipeHandleOut, string pipeHandleIn) => { // Validate args. ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut)); ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn)); // Start the child process. var assemblyDirectory = IOUtil.GetBinPath(); string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName); workerProcessTask = processInvoker.ExecuteAsync( workingDirectory: assemblyDirectory, fileName: workerFileName, arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn, environment: null, cancellationToken: workerProcessCancelTokenSource.Token); }); // Send the job request message. // Kill the worker process if sending the job message times out. The worker // process may have successfully received the job message. try { Trace.Info("Send job request message to worker."); using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout)) { await processChannel.SendAsync( messageType: MessageType.NewJobRequest, body: JsonUtility.ToString(message), cancellationToken: csSendJobRequest.Token); } } catch (OperationCanceledException) { // message send been cancelled. // timeout 45 sec. kill worker. Trace.Info("Job request message sending been cancelled, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { // worker process been killed. } Trace.Info("Stop renew job request."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; // not finish the job request since the job haven't run on worker at all, we will not going to set a result to server. return; } TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded; // wait for renewlock, worker process or cancellation token been fired. var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken)); if (completedTask == workerProcessTask) { // worker finished successfully, complete job request with result, stop renew lock, job has finished. int returnCode = await workerProcessTask; Trace.Info("Worker finished. Code: " + returnCode); TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode); Trace.Info($"finish job request with result: {result}"); term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result)); // complete job request await CompleteJobRequestAsync(_poolId, message, lockToken, result); Trace.Info("Stop renew job request."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; return; } else if (completedTask == renewJobRequest) { resultOnAbandonOrCancel = TaskResult.Abandoned; } else { resultOnAbandonOrCancel = TaskResult.Canceled; } // renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage) // cancel worker gracefully first, then kill it after 45 sec try { Trace.Info("Send job cancellation message to worker."); using (var csSendCancel = new CancellationTokenSource(ChannelTimeout)) { await processChannel.SendAsync( messageType: MessageType.CancelRequest, body: string.Empty, cancellationToken: csSendCancel.Token); } } catch (OperationCanceledException) { // message send been cancelled. Trace.Info("Job cancel message sending been cancelled, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { // worker process been killed. } } // wait worker to exit within 45 sec, then kill worker. using (var csKillWorker = new CancellationTokenSource(TimeSpan.FromSeconds(45))) { completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, csKillWorker.Token)); } // worker haven't exit within 45 sec. if (completedTask != workerProcessTask) { Trace.Info("worker process haven't exit after 45 sec, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { // worker process been killed. } } Trace.Info($"finish job request with result: {resultOnAbandonOrCancel}"); term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel)); // complete job request with cancel result, stop renew lock, job has finished. //TODO: don't finish job request on abandon await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel); Trace.Info("Stop renew job request."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; } } }
public void InitializeJob(JobRequestMessage message, CancellationToken token) { // Validation Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection)); ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); ArgUtil.NotNull(message.Plan, nameof(message.Plan)); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); // Features Features = ApiUtil.GetFeatures(message.Plan); // Endpoints Endpoints = message.Environment.Endpoints; Endpoints.Add(message.Environment.SystemConnection); // Variables (constructor performs initial recursive expansion) List <string> warnings; Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings); // Proxy variables var proxyConfiguration = HostContext.GetService <IProxyConfiguration>(); if (!string.IsNullOrEmpty(proxyConfiguration.ProxyUrl)) { Variables.Set(Constants.Variables.Agent.ProxyUrl, proxyConfiguration.ProxyUrl); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty); if (!string.IsNullOrEmpty(proxyConfiguration.ProxyUsername)) { Variables.Set(Constants.Variables.Agent.ProxyUsername, proxyConfiguration.ProxyUsername); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty); } if (!string.IsNullOrEmpty(proxyConfiguration.ProxyPassword)) { Variables.Set(Constants.Variables.Agent.ProxyPassword, proxyConfiguration.ProxyPassword, true); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty); } } // Job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, name: message.JobName, order: 1); // The job timeline record must be at order 1. // Logger (must be initialized before writing warnings). _logger = HostContext.CreateService <IPagingLogger>(); _logger.Setup(_mainTimelineId, _record.Id); // Log warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Verbosity (from system.debug). WriteDebug = Variables.System_Debug ?? false; // Hook up JobServerQueueThrottling event, we will log warning on server tarpit. _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; }
public abstract Task <int> HandleJobRequest(JobRequestMessage message);
public void InitializeJob(JobRequestMessage message, CancellationToken token) { // Validation Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection)); ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); ArgUtil.NotNull(message.Plan, nameof(message.Plan)); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); // Features Features = ApiUtil.GetFeatures(message.Plan); // Endpoints Endpoints = message.Environment.Endpoints; Endpoints.Add(message.Environment.SystemConnection); // SecureFiles SecureFiles = message.Environment.SecureFiles; // Variables (constructor performs initial recursive expansion) List <string> warnings; Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings); // Prepend Path PrependPath = new List <string>(); // Docker Container = new ContainerInfo() { ContainerImage = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"), ContainerName = $"VSTS_{Variables.System_HostType.ToString()}_{message.JobId.ToString("D")}", }; // Proxy variables var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>(); if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress)) { Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty); if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername)) { Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty); } if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword)) { Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty); } if (agentWebProxy.ProxyBypassList.Count > 0) { Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList)); } } // Job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, displayName: message.JobName, refName: message.JobRefName, order: null); // The job timeline record's order is set by server. // Logger (must be initialized before writing warnings). _logger = HostContext.CreateService <IPagingLogger>(); _logger.Setup(_mainTimelineId, _record.Id); // Log warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Verbosity (from system.debug). WriteDebug = Variables.System_Debug ?? false; // Hook up JobServerQueueThrottling event, we will log warning on server tarpit. _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; }
public async Task <TaskResult> RunAsync(JobRequestMessage message, CancellationToken jobRequestCancellationToken) { // Validate parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); ArgUtil.NotNull(message.Tasks, nameof(message.Tasks)); Trace.Info("Job ID {0}", message.JobId); if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) && StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken])) { // TODO: get access token use Util Method message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"]; } // Setup the job server and job server queue. var jobServer = HostContext.GetService <IJobServer>(); await jobServer.ConnectAsync(ApiUtil.GetVssConnection(message)); var jobServerQueue = HostContext.GetService <IJobServerQueue>(); jobServerQueue.Start(message); IExecutionContext jobContext = null; try { // Create the job execution context. jobContext = HostContext.CreateService <IExecutionContext>(); jobContext.InitializeJob(message, jobRequestCancellationToken); Trace.Info("Starting the job execution context."); jobContext.Start(); // Expand the endpoint data values. foreach (ServiceEndpoint endpoint in jobContext.Endpoints) { jobContext.Variables.ExpandValues(target: endpoint.Data); } // Get the job extensions. Trace.Info("Getting job extensions."); string hostType = jobContext.Variables.System_HostType; var extensionManager = HostContext.GetService <IExtensionManager>(); IJobExtension[] extensions = (extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>()) .Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase)) .ToArray(); // Add the prepare steps. Trace.Info("Adding job prepare extensions."); List <IStep> steps = new List <IStep>(); foreach (IJobExtension extension in extensions) { if (extension.PrepareStep != null) { Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}."); extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName); steps.Add(extension.PrepareStep); } } // Add the task steps. Trace.Info("Adding tasks."); foreach (TaskInstance taskInstance in message.Tasks) { Trace.Verbose($"Adding {taskInstance.DisplayName}."); var taskRunner = HostContext.CreateService <ITaskRunner>(); taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName); taskRunner.TaskInstance = taskInstance; steps.Add(taskRunner); } // Add the finally steps. Trace.Info("Adding job finally extensions."); foreach (IJobExtension extension in extensions) { if (extension.FinallyStep != null) { Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}."); extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName); steps.Add(extension.FinallyStep); } } // Download tasks if not already in the cache Trace.Info("Downloading task definitions."); var taskManager = HostContext.GetService <ITaskManager>(); try { await taskManager.DownloadAsync(jobContext, message.Tasks); } catch (OperationCanceledException ex) { // set the job to canceled Trace.Error($"Caught exception: {ex}"); jobContext.Error(ex); return(jobContext.Complete(TaskResult.Canceled)); } catch (Exception ex) { // Log the error and fail the job. Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}"); jobContext.Error(ex); return(jobContext.Complete(TaskResult.Failed)); } // Run the steps. var stepsRunner = HostContext.GetService <IStepsRunner>(); try { await stepsRunner.RunAsync(jobContext, steps); } catch (OperationCanceledException ex) { // set the job to canceled Trace.Error($"Caught exception: {ex}"); jobContext.Error(ex); return(jobContext.Complete(TaskResult.Canceled)); } catch (Exception ex) { // Log the error and fail the job. Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}"); jobContext.Error(ex); return(jobContext.Complete(TaskResult.Failed)); } Trace.Info($"Job result: {jobContext.Result}"); // Complete the job. Trace.Info("Completing the job execution context."); return(jobContext.Complete()); } finally { // Drain the job server queue. if (jobServerQueue != null) { try { Trace.Info("Shutting down the job server queue."); await jobServerQueue.ShutdownAsync(); } catch (Exception ex) { Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}"); } } } }
public async Task<TaskResult> RunAsync(JobRequestMessage message, CancellationToken jobRequestCancellationToken) { // Validate parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Environment, nameof(message.Environment)); ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables)); ArgUtil.NotNull(message.Tasks, nameof(message.Tasks)); Trace.Info("Job ID {0}", message.JobId); if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) && StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken])) { // TODO: get access token use Util Method message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"]; } // Setup the job server and job server queue. var jobServer = HostContext.GetService<IJobServer>(); var jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection); Uri jobServerUrl = ReplaceWithConfigUriBase(message.Environment.SystemConnection.Url); Trace.Info($"Creating job server with URL: {jobServerUrl}"); var jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential); await jobServer.ConnectAsync(jobConnection); var jobServerQueue = HostContext.GetService<IJobServerQueue>(); jobServerQueue.Start(message); IExecutionContext jobContext = null; try { // Create the job execution context. jobContext = HostContext.CreateService<IExecutionContext>(); jobContext.InitializeJob(message, jobRequestCancellationToken); Trace.Info("Starting the job execution context."); jobContext.Start(); // Set agent variables. AgentSettings settings = HostContext.GetService<IConfigurationStore>().GetSettings(); jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture)); jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath()); jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName); jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName); jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName); jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext)); jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), "vstsom")); jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext)); jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext)); // prefer task definitions url, then TFS collection url, then TFS account url var taskServer = HostContext.GetService<ITaskServer>(); Uri taskServerUri = null; if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri)) { taskServerUri = ReplaceWithConfigUriBase(new Uri(jobContext.Variables.System_TaskDefinitionsUri)); } else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl)) { taskServerUri = ReplaceWithConfigUriBase(new Uri(jobContext.Variables.System_TFCollectionUrl)); } var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection); if (taskServerUri != null) { Trace.Info($"Creating task server with {taskServerUri}"); await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential)); } if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist(jobRequestCancellationToken)) { Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist."); var configStore = HostContext.GetService<IConfigurationStore>(); taskServerUri = ReplaceWithConfigUriBase(new Uri(configStore.GetSettings().ServerUrl)); Trace.Info($"Recreate task server with configuration server url: {taskServerUri}"); await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential)); } // Expand the endpoint data values. foreach (ServiceEndpoint endpoint in jobContext.Endpoints) { jobContext.Variables.ExpandValues(target: endpoint.Data); VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data); } // Get the job extensions. Trace.Info("Getting job extensions."); string hostType = jobContext.Variables.System_HostType; var extensionManager = HostContext.GetService<IExtensionManager>(); IJobExtension[] extensions = (extensionManager.GetExtensions<IJobExtension>() ?? new List<IJobExtension>()) .Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase)) .ToArray(); // Add the prepare steps. Trace.Info("Adding job prepare extensions."); List<IStep> steps = new List<IStep>(); foreach (IJobExtension extension in extensions) { if (extension.PrepareStep != null) { Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}."); extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName); steps.Add(extension.PrepareStep); } } // Add the task steps. Trace.Info("Adding tasks."); foreach (TaskInstance taskInstance in message.Tasks) { Trace.Verbose($"Adding {taskInstance.DisplayName}."); var taskRunner = HostContext.CreateService<ITaskRunner>(); //// TODO: Presently 'TimeoutInMinutes' is not available in DistributedTask.WebApi TaskInstance. //// Next update it will be available. Once it is available, pass timeout to CreateChild. It will enable task timeout feature. //// var timeout = taskInstance.TimeoutInMinutes > 0 ? TimeSpan.FromMinutes(taskInstance.TimeoutInMinutes) : Timeout.InfiniteTimeSpan; taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName); taskRunner.TaskInstance = taskInstance; steps.Add(taskRunner); } // Add the finally steps. Trace.Info("Adding job finally extensions."); foreach (IJobExtension extension in extensions) { if (extension.FinallyStep != null) { Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}."); extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName); steps.Add(extension.FinallyStep); } } // Download tasks if not already in the cache Trace.Info("Downloading task definitions."); var taskManager = HostContext.GetService<ITaskManager>(); try { await taskManager.DownloadAsync(jobContext, message.Tasks); } catch (OperationCanceledException ex) { // set the job to canceled Trace.Error($"Caught exception: {ex}"); jobContext.Error(ex); return jobContext.Complete(TaskResult.Canceled); } catch (Exception ex) { // Log the error and fail the job. Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}"); jobContext.Error(ex); return jobContext.Complete(TaskResult.Failed); } // Run the steps. var stepsRunner = HostContext.GetService<IStepsRunner>(); try { await stepsRunner.RunAsync(jobContext, steps); } catch (OperationCanceledException ex) { // set the job to canceled Trace.Error($"Caught exception: {ex}"); jobContext.Error(ex); return jobContext.Complete(TaskResult.Canceled); } catch (Exception ex) { // Log the error and fail the job. Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}"); jobContext.Error(ex); return jobContext.Complete(TaskResult.Failed); } Trace.Info($"Job result: {jobContext.Result}"); // Complete the job. Trace.Info("Completing the job execution context."); return jobContext.Complete(); } finally { // Drain the job server queue. if (jobServerQueue != null) { try { Trace.Info("Shutting down the job server queue."); await jobServerQueue.ShutdownAsync(); } catch (Exception ex) { Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}"); } } } }
public async Task RunIPCEndToEnd() { using (var server = new ProcessChannel()) { JobRequestMessage result = null; TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; JobEnvironment environment = new JobEnvironment(); List <TaskInstance> tasks = new List <TaskInstance>(); Guid JobId = Guid.NewGuid(); var jobRequest = new JobRequestMessage(plan, timeline, JobId, "someJob", environment, tasks); Process jobProcess; server.StartServer((p1, p2) => { string clientFileName = $"Test{IOUtil.ExeExtension}"; jobProcess = new Process(); jobProcess.StartInfo.FileName = clientFileName; jobProcess.StartInfo.Arguments = "spawnclient " + p1 + " " + p2; jobProcess.EnableRaisingEvents = true; jobProcess.Start(); }); var cs = new CancellationTokenSource(); await server.SendAsync(MessageType.NewJobRequest, JsonUtility.ToString(jobRequest), cs.Token); var packetReceiveTask = server.ReceiveAsync(cs.Token); Task[] taskToWait = { packetReceiveTask, Task.Delay(5000) }; await Task.WhenAny(taskToWait); bool timedOut = !packetReceiveTask.IsCompleted; // Wait until response is received if (timedOut) { cs.Cancel(); try { await packetReceiveTask; } catch (OperationCanceledException) { // Ignore OperationCanceledException and TaskCanceledException exceptions } catch (AggregateException errors) { // Ignore OperationCanceledException and TaskCanceledException exceptions errors.Handle(e => e is OperationCanceledException); } } else { result = JsonUtility.FromString <JobRequestMessage>(packetReceiveTask.Result.Body); } // Wait until response is received if (timedOut) { Assert.True(false, "Test timed out."); } else { Assert.True(jobRequest.JobId.Equals(result.JobId) && jobRequest.JobName.Equals(result.JobName)); } } }
public async Task RunIPCEndToEnd() { using (var server = new ProcessChannel()) { JobRequestMessage result = null; TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; JobEnvironment environment = new JobEnvironment(); List<TaskInstance> tasks = new List<TaskInstance>(); Guid JobId = Guid.NewGuid(); var jobRequest = new JobRequestMessage(plan, timeline, JobId, "someJob", environment, tasks); Process jobProcess; server.StartServer((p1, p2) => { string clientFileName = $"Test{IOUtil.ExeExtension}"; jobProcess = new Process(); jobProcess.StartInfo.FileName = clientFileName; jobProcess.StartInfo.Arguments = "spawnclient " + p1 + " " + p2; jobProcess.EnableRaisingEvents = true; jobProcess.Start(); }); var cs = new CancellationTokenSource(); await server.SendAsync(MessageType.NewJobRequest, JsonUtility.ToString(jobRequest), cs.Token); var packetReceiveTask = server.ReceiveAsync(cs.Token); Task[] taskToWait = { packetReceiveTask, Task.Delay(30*1000) }; await Task.WhenAny(taskToWait); bool timedOut = !packetReceiveTask.IsCompleted; // Wait until response is received if (timedOut) { cs.Cancel(); try { await packetReceiveTask; } catch (OperationCanceledException) { // Ignore OperationCanceledException and TaskCanceledException exceptions } catch (AggregateException errors) { // Ignore OperationCanceledException and TaskCanceledException exceptions errors.Handle(e => e is OperationCanceledException); } } else { result = JsonUtility.FromString<JobRequestMessage>(packetReceiveTask.Result.Body); } // Wait until response is received if (timedOut) { Assert.True(false, "Test timed out."); } else { Assert.True(jobRequest.JobId.Equals(result.JobId) && jobRequest.JobName.Equals(result.JobName)); } } }