private void InitializeSecretMasker(JobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
var secretMasker = HostContext.GetService <ISecretMasker>();
// Add mask hints
var variables = message?.Environment?.Variables ?? new Dictionary <string, string>();
foreach (MaskHint maskHint in (message.Environment.MaskHints ?? new List <MaskHint>()))
{
if (maskHint.Type == MaskType.Regex)
{
secretMasker.AddRegex(maskHint.Value);
// Also add the JSON escaped string since the job message is traced in the diag log.
secretMasker.AddValue(JsonConvert.ToString(maskHint.Value ?? string.Empty));
}
else if (maskHint.Type == MaskType.Variable)
{
string value;
if (variables.TryGetValue(maskHint.Value, out value) &&
!string.IsNullOrEmpty(value))
{
secretMasker.AddValue(value);
// Also add the JSON escaped string since the job message is traced in the diag log.
secretMasker.AddValue(JsonConvert.ToString(value));
}
}
else
{
// TODO: Should we fail instead? Do any additional pains need to be taken here? Should the job message not be traced?
Trace.Warning($"Unsupported mask type '{maskHint.Type}'.");
}
}
// TODO: Avoid adding redundant secrets. If the endpoint auth matches the system connection, then it's added as a value secret and as a regex secret. Once as a value secret b/c of the following code that iterates over each endpoint. Once as a regex secret due to the hint sent down in the job message.
// Add masks for service endpoints
foreach (ServiceEndpoint endpoint in message.Environment.Endpoints ?? new List <ServiceEndpoint>())
{
foreach (string value in endpoint.Authorization?.Parameters?.Values ?? new string[0])
{
if (!string.IsNullOrEmpty(value))
{
secretMasker.AddValue(value);
// This is precautionary if the secret is used in an URL. For example, if "allow scripts
// access to OAuth token" is checked, then the repository auth key is injected into the
// URL for a Git repository's remote configuration.
if (!Uri.EscapeDataString(value).Equals(value, StringComparison.OrdinalIgnoreCase))
{
secretMasker.AddValue(Uri.EscapeDataString(value));
}
}
}
}
}
public void InitializeJob(JobRequestMessage message, CancellationToken token)
{
// Validate/store parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection));
ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
CancellationToken = token;
// Initialize the environment.
Endpoints = message.Environment.Endpoints;
// add system connect to the endpoint list.
Endpoints.Add(message.Environment.SystemConnection);
List <string> warnings;
Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings);
// Initialize the job timeline record.
// the job timeline record is at order 1.
InitializeTimelineRecord(message.Timeline.Id, message.JobId, null, ExecutionContextType.Job, message.JobName, 1);
// Log any warnings.
foreach (string warning in (warnings ?? new List <string>()))
{
this.Warning(warning);
}
// Set system.debug
WriteDebug = Variables.System_Debug ?? false;
}
private void InitializeSecretMasker(JobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
var secretMasker = HostContext.GetService<ISecretMasker>();
// Add mask hints
var variables = message?.Environment?.Variables ?? new Dictionary<string, string>();
foreach (MaskHint maskHint in (message.Environment.MaskHints ?? new List<MaskHint>()))
{
if (maskHint.Type == MaskType.Regex)
{
secretMasker.AddRegex(maskHint.Value);
// Also add the JSON escaped string since the job message is traced in the diag log.
secretMasker.AddValue(JsonConvert.ToString(maskHint.Value ?? string.Empty));
}
else if (maskHint.Type == MaskType.Variable)
{
string value;
if (variables.TryGetValue(maskHint.Value, out value) &&
!string.IsNullOrEmpty(value))
{
secretMasker.AddValue(value);
// Also add the JSON escaped string since the job message is traced in the diag log.
secretMasker.AddValue(JsonConvert.ToString(value));
}
}
else
{
// TODO: Should we fail instead? Do any additional pains need to be taken here? Should the job message not be traced?
Trace.Warning($"Unsupported mask type '{maskHint.Type}'.");
}
}
// TODO: Avoid adding redundant secrets. If the endpoint auth matches the system connection, then it's added as a value secret and as a regex secret. Once as a value secret b/c of the following code that iterates over each endpoint. Once as a regex secret due to the hint sent down in the job message.
// Add masks for service endpoints
foreach (ServiceEndpoint endpoint in message.Environment.Endpoints ?? new List<ServiceEndpoint>())
{
foreach (string value in endpoint.Authorization?.Parameters?.Values ?? new string[0])
{
secretMasker.AddValue(value);
// This is precautionary if the secret is used in an URL. For example, if "allow scripts
// access to OAuth token" is checked, then the repository auth key is injected into the
// URL for a Git repository's remote configuration.
if (!Uri.EscapeDataString(value).Equals(value, StringComparison.OrdinalIgnoreCase))
{
secretMasker.AddValue(Uri.EscapeDataString(value));
}
}
}
}
private JobRequestMessage CreateJobRequestMessage()
{
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = null;
JobEnvironment environment = new JobEnvironment();
List<TaskInstance> tasks = new List<TaskInstance>();
Guid JobId = Guid.NewGuid();
var jobRequest = new JobRequestMessage(plan, timeline, JobId, "someJob", environment, tasks);
return jobRequest;
}
private JobRequestMessage CreateJobRequestMessage(string jobName)
{
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = null;
JobEnvironment environment = new JobEnvironment();
List <TaskInstance> tasks = new List <TaskInstance>();
Guid JobId = Guid.NewGuid();
var jobRequest = new JobRequestMessage(plan, timeline, JobId, jobName, environment, tasks);
return(jobRequest);
}
public override async Task <int> HandleJobRequest(JobRequestMessage message)
{
var submission = await Context.Submissions.FindAsync(message.TargetId);
if (submission is null || submission.RequestVersion >= message.RequestVersion)
{
Logger.LogDebug($"IgnoreJudgeRequestMessage" +
$" SubmissionId={message.TargetId}" +
$" RequestVersion={message.RequestVersion}");
return(0);
}
public override async Task <int> HandleJobRequest(JobRequestMessage message)
{
var plagiarism = await Context.Plagiarisms.FindAsync(message.TargetId);
if (plagiarism is null || plagiarism.RequestVersion >= message.RequestVersion)
{
Logger.LogDebug($"IgnoreCheckPlagiarismMessage" +
$" PlagiarismId={message.TargetId}" +
$" RequestVersion={message.RequestVersion}");
return(0);
}
private JobRequestMessage CreateJobRequestMessage(string jobName)
{
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = null;
JobEnvironment environment = new JobEnvironment();
environment.Variables[Constants.Variables.System.Culture] = "en-US";
List<TaskInstance> tasks = new List<TaskInstance>();
Guid JobId = Guid.NewGuid();
var jobRequest = new JobRequestMessage(plan, timeline, JobId, jobName, environment, tasks);
return jobRequest;
}
public void TestRun()
{
using (var hc = new TestHostContext(this))
using (var workerManager = new WorkerManager())
{
hc.EnqueueInstance <IJobDispatcher>(_jobDispatcher.Object);
workerManager.Initialize(hc);
JobRequestMessage jobMessage = CreateJobRequestMessage();
_jobDispatcher.Setup(x => x.RunAsync(jobMessage, It.IsAny <CancellationToken>()))
.Returns(Task.FromResult <int>(21));
workerManager.Run(jobMessage);
_jobDispatcher.Verify(x => x.RunAsync(jobMessage, It.IsAny <CancellationToken>()),
"IJobDispatcher.RunAsync not invoked");
}
}
private void SetCulture(JobRequestMessage message)
{
// Extract the culture name from the job's variable dictionary.
// The variable does not exist for TFS 2015 RTM and Update 1.
// It was introduced in Update 2.
string culture;
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
if (message.Environment.Variables.TryGetValue(Constants.Variables.System.Culture, out culture))
{
// Set the default thread culture.
HostContext.SetDefaultCulture(culture);
}
}
public void Run(JobRequestMessage jobRequestMessage)
{
Trace.Info("Job request {0} received.", jobRequestMessage.JobId);
var jobDispatcher = HostContext.CreateService <IJobDispatcher>();
Task <int> jobDispatcherTask;
var cancellationTokenSource = new CancellationTokenSource();
jobDispatcherTask = jobDispatcher.RunAsync(jobRequestMessage, cancellationTokenSource.Token);
var postJobDispatcherAction = new Action <Task <int>, Object>((task, obj) =>
{
lock (_lock)
{
//protect from running after WorkerManager has been disposed
if ((obj as WorkerManager)._disposed)
{
return;
}
if (task.Status == TaskStatus.Canceled)
{
Trace.Info("Job request {0} was canceled.", jobRequestMessage.JobId);
}
else if (task.Status == TaskStatus.Faulted)
{
Trace.Error("Job request {0} failed witn an exception.", jobRequestMessage.JobId);
Trace.Error(task.Exception);
}
else
{
Trace.Info("Job request {0} processed with return code {1}.", jobRequestMessage.JobId, task.Result);
}
JobDispatcherItem deletedJob;
if (_jobsInProgress.TryGetValue(jobRequestMessage.JobId, out deletedJob))
{
_jobsInProgress.Remove(jobRequestMessage.JobId);
deletedJob.Dispose();
}
}
});
lock (_lock)
{
var jobItem = new JobDispatcherItem(jobDispatcher, null, cancellationTokenSource);
// store the new job item, so that it can be cancelled later if needed
_jobsInProgress[jobRequestMessage.JobId] = jobItem;
//run code after the job completes, which prints the result int the trace
jobItem.DispatcherTask = jobDispatcherTask.ContinueWith(postJobDispatcherAction, this);
}
}
public async void DispatchesJobRequest()
{
//Arrange
using (var hc = new TestHostContext(this))
{
var jobDispatcher = new JobDispatcher();
hc.SetSingleton <IConfigurationStore>(_configurationStore.Object);
hc.SetSingleton <IAgentServer>(_agentServer.Object);
hc.EnqueueInstance <IProcessChannel>(_processChannel.Object);
hc.EnqueueInstance <IProcessInvoker>(_processInvoker.Object);
_configurationStore.Setup(x => x.GetSettings()).Returns(new AgentSettings()
{
PoolId = 1
});
jobDispatcher.Initialize(hc);
var ts = new CancellationTokenSource();
JobRequestMessage message = CreateJobRequestMessage();
string strMessage = JsonUtility.ToString(message);
_processInvoker.Setup(x => x.ExecuteAsync(It.IsAny <String>(), It.IsAny <String>(), "spawnclient 1 2", null, It.IsAny <CancellationToken>()))
.Returns(Task.FromResult <int>(56));
_processChannel.Setup(x => x.StartServer(It.IsAny <StartProcessDelegate>()))
.Callback((StartProcessDelegate startDel) => { startDel("1", "2"); });
_processChannel.Setup(x => x.SendAsync(MessageType.NewJobRequest, It.Is <string>(s => s.Equals(strMessage)), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask);
var request = new TaskAgentJobRequest();
PropertyInfo sessionIdProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(sessionIdProperty);
sessionIdProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5));
_agentServer.Setup(x => x.RenewAgentRequestAsync(It.IsAny <int>(), It.IsAny <long>(), It.IsAny <Guid>(), It.IsAny <CancellationToken>())).Returns(Task.FromResult <TaskAgentJobRequest>(request));
_agentServer.Setup(x => x.FinishAgentRequestAsync(It.IsAny <int>(), It.IsAny <long>(), It.IsAny <Guid>(), It.IsAny <DateTime>(), It.IsAny <TaskResult>(), It.IsAny <CancellationToken>())).Returns(Task.FromResult <TaskAgentJobRequest>(new TaskAgentJobRequest()));
//Actt
jobDispatcher.Run(message);
//Assert
await jobDispatcher.WaitAsync(CancellationToken.None);
}
}
private void SetCulture(JobRequestMessage message)
{
// Extract the culture name from the job's variable dictionary.
string culture;
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
if (!message.Environment.Variables.TryGetValue(Constants.Variables.System.Culture, out culture))
{
culture = null;
}
// Set the default thread culture.
// TODO: Does this validation need to be removed? The variable does not exist prior to 2015 Update 2.
ArgUtil.NotNullOrEmpty(culture, nameof(culture));
HostContext.SetDefaultCulture(culture);
}
public void InitializeJob(JobRequestMessage message, CancellationToken token)
{
// Validate/store parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection));
ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Initialize the environment.
Endpoints = message.Environment.Endpoints;
// Add the system connection to the endpoint list.
Endpoints.Add(message.Environment.SystemConnection);
// Initialize the variables. The constructor handles the initial recursive expansion.
List <string> warnings;
Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings);
// Initialize the job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
name: message.JobName,
order: 1); // The job timeline record must be at order 1.
// Initialize the logger before writing warnings.
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log any warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Initialize the verbosity (based on system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public async void TestCancel()
{
//Arrange
using (var hc = new TestHostContext(this))
using (var workerManager = new WorkerManager())
{
hc.EnqueueInstance <IJobDispatcher>(_jobDispatcher.Object);
workerManager.Initialize(hc);
JobRequestMessage jobMessage = CreateJobRequestMessage();
JobCancelMessage cancelMessage = CreateJobCancelMessage(jobMessage.JobId);
bool started = false;
Task jobTask = null;
_jobDispatcher.Setup(x => x.RunAsync(jobMessage, It.IsAny <CancellationToken>()))
.Returns(async(JobRequestMessage message, CancellationToken token) =>
{
jobTask = Task.Delay(5000, token);
started = true;
await jobTask;
return(0);
});
workerManager.Run(jobMessage);
int i = 20;
while (i > 0 && (!started))
{
await Task.Delay(10);
i--;
}
Assert.True(started);
//Act
//send cancel message
workerManager.Cancel(cancelMessage);
//Assert
//wait up to 2 sec for cancellation to be processed
Task[] taskToWait = { jobTask, Task.Delay(2000) };
await Task.WhenAny(taskToWait);
_jobDispatcher.Verify(x => x.RunAsync(jobMessage, It.IsAny <CancellationToken>()),
$"{nameof(_jobDispatcher.Object.RunAsync)} not invoked");
Assert.True(jobTask.IsCompleted);
Assert.True(jobTask.IsCanceled);
}
}
// The server only send down OAuth token in Job Request message.
public static VssConnection GetVssConnection(JobRequestMessage jobRequest)
{
ArgUtil.NotNull(jobRequest, nameof(jobRequest));
ArgUtil.NotNull(jobRequest.Environment, nameof(jobRequest.Environment));
ArgUtil.NotNull(jobRequest.Environment.SystemConnection, nameof(jobRequest.Environment.SystemConnection));
ArgUtil.NotNull(jobRequest.Environment.SystemConnection.Url, nameof(jobRequest.Environment.SystemConnection.Url));
Uri serverUrl = jobRequest.Environment.SystemConnection.Url;
var credentials = GetVssCredential(jobRequest.Environment.SystemConnection);
if (credentials == null)
{
throw new ArgumentNullException(nameof(credentials));
}
else
{
return(CreateConnection(serverUrl, credentials));
}
}
// The server only send down OAuth token in Job Request message.
public static VssConnection GetVssConnection(JobRequestMessage jobRequest)
{
ArgUtil.NotNull(jobRequest, nameof(jobRequest));
ArgUtil.NotNull(jobRequest.Environment, nameof(jobRequest.Environment));
ArgUtil.NotNull(jobRequest.Environment.SystemConnection, nameof(jobRequest.Environment.SystemConnection));
ArgUtil.NotNull(jobRequest.Environment.SystemConnection.Url, nameof(jobRequest.Environment.SystemConnection.Url));
Uri serverUrl = jobRequest.Environment.SystemConnection.Url;
var credentials = GetVssCredential(jobRequest.Environment.SystemConnection);
if (credentials == null)
{
throw new ArgumentNullException(nameof(credentials));
}
else
{
return CreateConnection(serverUrl, credentials);
}
}
public void Start(JobRequestMessage jobRequest)
{
Trace.Entering();
if (HostContext.RunMode == RunMode.Local)
{
_term = HostContext.GetService <ITerminal>();
return;
}
if (_queueInProcess)
{
Trace.Info("No-opt, all queue process tasks are running.");
return;
}
ArgUtil.NotNull(jobRequest, nameof(jobRequest));
ArgUtil.NotNull(jobRequest.Plan, nameof(jobRequest.Plan));
ArgUtil.NotNull(jobRequest.Timeline, nameof(jobRequest.Timeline));
_scopeIdentifier = jobRequest.Plan.ScopeIdentifier;
_hubName = jobRequest.Plan.PlanType;
_planId = jobRequest.Plan.PlanId;
_jobTimelineId = jobRequest.Timeline.Id;
_jobTimelineRecordId = jobRequest.JobId;
// Server already create the job timeline
_timelineUpdateQueue[_jobTimelineId] = new ConcurrentQueue <TimelineRecord>();
_allTimelines.Add(_jobTimelineId);
// Start three dequeue task
Trace.Info("Start process web console line queue.");
_webConsoleLineDequeueTask = ProcessWebConsoleLinesQueueAsync();
Trace.Info("Start process file upload queue.");
_fileUploadDequeueTask = ProcessFilesUploadQueueAsync();
Trace.Info("Start process timeline update queue.");
_timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync();
_allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask };
_queueInProcess = true;
}
public void Run(JobRequestMessage jobRequestMessage)
{
Trace.Info($"Job request {jobRequestMessage.JobId} received.");
WorkerDispatcher currentDispatch = null;
if (_jobDispatchedQueue.Count > 0)
{
Guid dispatchedJobId = _jobDispatchedQueue.Dequeue();
if (_jobInfos.TryGetValue(dispatchedJobId, out currentDispatch))
{
Trace.Verbose($"Retrive previous WorkerDispather for job {currentDispatch.JobId}.");
}
}
WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token);
_jobInfos.TryAdd(newDispatch.JobId, newDispatch);
_jobDispatchedQueue.Enqueue(newDispatch.JobId);
}
public void Run(JobRequestMessage jobRequestMessage)
{
Trace.Info($"Job request {jobRequestMessage.JobId} received.");
WorkerDispatcher currentDispatch = null;
if (_jobDispatchedQueue.Count > 0)
{
Guid dispatchedJobId = _jobDispatchedQueue.Dequeue();
if (_jobInfos.TryGetValue(dispatchedJobId, out currentDispatch))
{
Trace.Verbose($"Retrive previous WorkerDispather for job {currentDispatch.JobId}.");
}
}
WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token);
_jobInfos.TryAdd(newDispatch.JobId, newDispatch);
_jobDispatchedQueue.Enqueue(newDispatch.JobId);
}
public void InitializeJob(JobRequestMessage message, CancellationToken token)
{
// Validate/store parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection));
ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
CancellationToken = token;
// Initialize the environment.
Endpoints = message.Environment.Endpoints;
// Add the system connection to the endpoint list.
Endpoints.Add(message.Environment.SystemConnection);
// Initialize the variables. The constructor handles the initial recursive expansion.
List <string> warnings;
Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings);
// Initialize the job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
name: message.JobName,
order: 1); // The job timeline record must be at order 1.
// Log any warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Initialize the verbosity (based on system.debug).
WriteDebug = Variables.System_Debug ?? false;
}
private async Task CompleteJobRequestAsync(int poolId, JobRequestMessage message, Guid lockToken, TaskResult result)
{
var agentServer = HostContext.GetService <IAgentServer>();
try
{
using (var csFinishRequest = new CancellationTokenSource(ChannelTimeout))
{
await agentServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, csFinishRequest.Token);
}
}
catch (TaskAgentJobNotFoundException)
{
Trace.Info("TaskAgentJobNotFoundException received, job is no longer valid.");
}
catch (TaskAgentJobTokenExpiredException)
{
Trace.Info("TaskAgentJobTokenExpiredException received, job is no longer valid.");
}
// This should be the last thing to run so we don't notify external parties until actually finished
var notification = HostContext.GetService <IJobNotification>();
await notification.JobCompleted(message.JobId);
}
public void Start(JobRequestMessage jobRequest)
{
Trace.Entering();
if (_queueInProcess)
{
Trace.Info("No-opt, all queue process tasks are running.");
return;
}
ArgUtil.NotNull(jobRequest, nameof(jobRequest));
ArgUtil.NotNull(jobRequest.Plan, nameof(jobRequest.Plan));
ArgUtil.NotNull(jobRequest.Timeline, nameof(jobRequest.Timeline));
_scopeIdentifier = jobRequest.Plan.ScopeIdentifier;
_hubName = jobRequest.Plan.PlanType;
_planId = jobRequest.Plan.PlanId;
_jobTimelineId = jobRequest.Timeline.Id;
_jobTimelineRecordId = jobRequest.JobId;
// Server already create the job timeline
_timelineUpdateQueue[_jobTimelineId] = new ConcurrentQueue<TimelineRecord>();
_allTimelines.Add(_jobTimelineId);
// Start three dequeue task
Trace.Info("Start process web console line queue.");
_webConsoleLineDequeueTask = ProcessWebConsoleLinesQueueAsync();
Trace.Info("Start process file upload queue.");
_fileUploadDequeueTask = ProcessFilesUploadQueueAsync();
Trace.Info("Start process timeline update queue.");
_timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync();
_allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask };
_queueInProcess = true;
}
public async Task <bool> SendAsync(JobRequestMessage message)
{
var serialized = JsonConvert.SerializeObject(message);
var body = Encoding.UTF8.GetBytes(serialized);
using var scope = _factory.CreateScope();
var queueStatisticsService = scope.ServiceProvider.GetRequiredService <QueueStatisticsService>();
try
{
Channel.BasicPublish("", Queue, null, body);
await queueStatisticsService.AddJobRequestAsync(message);
Logger.LogDebug($"SendJobRequestMessage JobType={message.JobType}" +
$" TargetId={message.TargetId} RequestVersion={message.RequestVersion}");
return(true);
}
catch (Exception e)
{
Logger.LogError($"SendJobRequestMessage failed: {e.Message}");
Logger.LogError($"Stacktrace: {e.StackTrace}");
return(false);
}
}
private void _read()
{
_protcolId = m_io.ReadU1();
_messageType = ((S7MessageType)m_io.ReadU1());
_reserved = m_io.ReadU2be();
_pduReference = m_io.ReadU2be();
_parametersLength = m_io.ReadU2be();
_dataLength = m_io.ReadU2be();
if (MessageType == S7MessageType.AckData)
{
_error = new ResponseError(m_io, this, m_root);
}
switch (MessageType)
{
case S7MessageType.JobRequest: {
__raw_message = m_io.ReadBytes((ParametersLength + DataLength));
var io___raw_message = new KaitaiStream(__raw_message);
_message = new JobRequestMessage(io___raw_message, this, m_root);
break;
}
case S7MessageType.AckData: {
__raw_message = m_io.ReadBytes((ParametersLength + DataLength));
var io___raw_message = new KaitaiStream(__raw_message);
_message = new AckDataMessage(io___raw_message, this, m_root);
break;
}
default: {
__raw_message = m_io.ReadBytes((ParametersLength + DataLength));
var io___raw_message = new KaitaiStream(__raw_message);
_message = new MessageOther(io___raw_message, this, m_root);
break;
}
}
}
public async Task <TaskResult> RunAsync(JobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
Trace.Info("Job ID {0}", message.JobId);
if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
{
// TODO: get access token use Util Method
message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
}
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
var jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
var jobConnection = ApiUtil.CreateConnection(ReplaceWithConfigUriBase(message.Environment.SystemConnection.Url), jobServerCredential);
await jobServer.ConnectAsync(jobConnection);
var jobServerQueue = HostContext.GetService <IJobServerQueue>();
jobServerQueue.Start(message);
IExecutionContext jobContext = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath());
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), "vstsom"));
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
// prefer task definitions url, then TFS url
var taskServer = HostContext.GetService <ITaskServer>();
string taskUrl = jobContext.Variables.System_TaskDefinitionsUri;
Uri taskServerUri;
if (string.IsNullOrEmpty(taskUrl))
{
taskServerUri = ReplaceWithConfigUriBase(message.Environment.SystemConnection.Url);
Trace.Info($"Creating task server with tfs server url {taskServerUri.ToString()}");
}
else
{
taskServerUri = ReplaceWithConfigUriBase(new Uri(taskUrl));
Trace.Info($"Creating task server with {taskServerUri.ToString()}");
}
var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extensions.
Trace.Info("Getting job extensions.");
string hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
IJobExtension[] extensions =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase))
.ToArray();
// Add the prepare steps.
Trace.Info("Adding job prepare extensions.");
List <IStep> steps = new List <IStep>();
foreach (IJobExtension extension in extensions)
{
if (extension.PrepareStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}.");
extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName);
steps.Add(extension.PrepareStep);
}
}
// Add the task steps.
Trace.Info("Adding tasks.");
foreach (TaskInstance taskInstance in message.Tasks)
{
Trace.Verbose($"Adding {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName);
taskRunner.TaskInstance = taskInstance;
steps.Add(taskRunner);
}
// Add the finally steps.
Trace.Info("Adding job finally extensions.");
foreach (IJobExtension extension in extensions)
{
if (extension.FinallyStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}.");
extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName);
steps.Add(extension.FinallyStep);
}
}
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
try
{
await taskManager.DownloadAsync(jobContext, message.Tasks);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
// Run the steps.
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, steps);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
Trace.Info($"Job result: {jobContext.Result}");
// Complete the job.
Trace.Info("Completing the job execution context.");
return(jobContext.Complete());
}
finally
{
// Drain the job server queue.
if (jobServerQueue != null)
{
try
{
Trace.Info("Shutting down the job server queue.");
await jobServerQueue.ShutdownAsync();
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}");
}
}
}
}
private void SetCulture(JobRequestMessage message)
{
// Extract the culture name from the job's variable dictionary.
// The variable does not exist for TFS 2015 RTM and Update 1.
// It was introduced in Update 2.
string culture;
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
if (message.Environment.Variables.TryGetValue(Constants.Variables.System.Culture, out culture))
{
// Set the default thread culture.
HostContext.SetDefaultCulture(culture);
}
}
private async Task RunAsync(JobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = message.LockToken;
// start renew job request
Trace.Info("Start renew job request.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
// TODO: not need to return anything.
Trace.Info("Unable to renew job request for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
await CompleteJobRequestAsync(_poolId, requestId, lockToken, TaskResult.Canceled);
return;
}
Task <int> workerProcessTask = null;
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
// Start the child process.
var assemblyDirectory = IOUtil.GetBinPath();
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info("Send job request message to worker.");
using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 45 sec. kill worker.
Trace.Info("Job request message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info("Worker finished. Code: " + returnCode);
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result));
// complete job request
await CompleteJobRequestAsync(_poolId, requestId, lockToken, result);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after 45 sec
try
{
Trace.Info("Send job cancellation message to worker.");
using (var csSendCancel = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.CancelRequest,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info("Job cancel message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
// wait worker to exit within 45 sec, then kill worker.
using (var csKillWorker = new CancellationTokenSource(TimeSpan.FromSeconds(45)))
{
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, csKillWorker.Token));
}
// worker haven't exit within 45 sec.
if (completedTask != workerProcessTask)
{
Trace.Info("worker process haven't exit after 45 sec, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
Trace.Info($"finish job request with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
//TODO: don't finish job request on abandon
await CompleteJobRequestAsync(_poolId, requestId, lockToken, resultOnAbandonOrCancel);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
}
}
}
public void InitializeJob(JobRequestMessage message, CancellationToken token)
{
// Validate/store parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection));
ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
CancellationToken = token;
// Initialize the environment.
Endpoints = message.Environment.Endpoints;
// Add the system connection to the endpoint list.
Endpoints.Add(message.Environment.SystemConnection);
// Initialize the variables. The constructor handles the initial recursive expansion.
List<string> warnings;
Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings);
// Initialize the job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
name: message.JobName,
order: 1); // The job timeline record must be at order 1.
// Log any warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Initialize the verbosity (based on system.debug).
WriteDebug = Variables.System_Debug ?? false;
}
private async Task CompleteJobRequestAsync(int poolId, JobRequestMessage message, Guid lockToken, TaskResult result)
{
var agentServer = HostContext.GetService<IAgentServer>();
try
{
using (var csFinishRequest = new CancellationTokenSource(ChannelTimeout))
{
await agentServer.FinishAgentRequestAsync(poolId, message.RequestId, lockToken, DateTime.UtcNow, result, csFinishRequest.Token);
}
}
catch (TaskAgentJobNotFoundException)
{
Trace.Info("TaskAgentJobNotFoundException received, job is no longer valid.");
}
catch (TaskAgentJobTokenExpiredException)
{
Trace.Info("TaskAgentJobTokenExpiredException received, job is no longer valid.");
}
// This should be the last thing to run so we don't notify external parties until actually finished
var notification = HostContext.GetService<IJobNotification>();
await notification.JobCompleted(message.JobId);
}
private async Task RunAsync(JobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService<ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName));
// first job request renew succeed.
TaskCompletionSource<int> firstJobRequestRenewed = new TaskCompletionSource<int>();
var notification = HostContext.GetService<IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
await notification.JobStarted(message.JobId);
long requestId = message.RequestId;
Guid lockToken = message.LockToken;
// start renew job request
Trace.Info("Start renew job request.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
// TODO: not need to return anything.
Trace.Info("Unable to renew job request for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
Task<int> workerProcessTask = null;
using (var processChannel = HostContext.CreateService<IProcessChannel>())
using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
// Start the child process.
var assemblyDirectory = IOUtil.GetBinPath();
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info("Send job request message to worker.");
using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType: MessageType.NewJobRequest,
body: JsonUtility.ToString(message),
cancellationToken: csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 45 sec. kill worker.
Trace.Info("Job request message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info("Worker finished. Code: " + returnCode);
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result));
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after 45 sec
try
{
Trace.Info("Send job cancellation message to worker.");
using (var csSendCancel = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType: MessageType.CancelRequest,
body: string.Empty,
cancellationToken: csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info("Job cancel message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
// wait worker to exit within 45 sec, then kill worker.
using (var csKillWorker = new CancellationTokenSource(TimeSpan.FromSeconds(45)))
{
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, csKillWorker.Token));
}
// worker haven't exit within 45 sec.
if (completedTask != workerProcessTask)
{
Trace.Info("worker process haven't exit after 45 sec, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
Trace.Info($"finish job request with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
//TODO: don't finish job request on abandon
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
}
}
}
public void InitializeJob(JobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection));
ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = ApiUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Environment.Endpoints;
Endpoints.Add(message.Environment.SystemConnection);
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings);
// Proxy variables
var proxyConfiguration = HostContext.GetService <IProxyConfiguration>();
if (!string.IsNullOrEmpty(proxyConfiguration.ProxyUrl))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, proxyConfiguration.ProxyUrl);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(proxyConfiguration.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, proxyConfiguration.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(proxyConfiguration.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, proxyConfiguration.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
name: message.JobName,
order: 1); // The job timeline record must be at order 1.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public abstract Task <int> HandleJobRequest(JobRequestMessage message);
public void InitializeJob(JobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection));
ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = ApiUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Environment.Endpoints;
Endpoints.Add(message.Environment.SystemConnection);
// SecureFiles
SecureFiles = message.Environment.SecureFiles;
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings);
// Prepend Path
PrependPath = new List <string>();
// Docker
Container = new ContainerInfo()
{
ContainerImage = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"),
ContainerName = $"VSTS_{Variables.System_HostType.ToString()}_{message.JobId.ToString("D")}",
};
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
if (agentWebProxy.ProxyBypassList.Count > 0)
{
Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList));
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobName,
refName: message.JobRefName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public async Task <TaskResult> RunAsync(JobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
Trace.Info("Job ID {0}", message.JobId);
if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
{
// TODO: get access token use Util Method
message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
}
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
await jobServer.ConnectAsync(ApiUtil.GetVssConnection(message));
var jobServerQueue = HostContext.GetService <IJobServerQueue>();
jobServerQueue.Start(message);
IExecutionContext jobContext = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
}
// Get the job extensions.
Trace.Info("Getting job extensions.");
string hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
IJobExtension[] extensions =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase))
.ToArray();
// Add the prepare steps.
Trace.Info("Adding job prepare extensions.");
List <IStep> steps = new List <IStep>();
foreach (IJobExtension extension in extensions)
{
if (extension.PrepareStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}.");
extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName);
steps.Add(extension.PrepareStep);
}
}
// Add the task steps.
Trace.Info("Adding tasks.");
foreach (TaskInstance taskInstance in message.Tasks)
{
Trace.Verbose($"Adding {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName);
taskRunner.TaskInstance = taskInstance;
steps.Add(taskRunner);
}
// Add the finally steps.
Trace.Info("Adding job finally extensions.");
foreach (IJobExtension extension in extensions)
{
if (extension.FinallyStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}.");
extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName);
steps.Add(extension.FinallyStep);
}
}
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
try
{
await taskManager.DownloadAsync(jobContext, message.Tasks);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
// Run the steps.
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, steps);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
Trace.Info($"Job result: {jobContext.Result}");
// Complete the job.
Trace.Info("Completing the job execution context.");
return(jobContext.Complete());
}
finally
{
// Drain the job server queue.
if (jobServerQueue != null)
{
try
{
Trace.Info("Shutting down the job server queue.");
await jobServerQueue.ShutdownAsync();
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}");
}
}
}
}
public async Task<TaskResult> RunAsync(JobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
Trace.Info("Job ID {0}", message.JobId);
if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
{
// TODO: get access token use Util Method
message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
}
// Setup the job server and job server queue.
var jobServer = HostContext.GetService<IJobServer>();
var jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
Uri jobServerUrl = ReplaceWithConfigUriBase(message.Environment.SystemConnection.Url);
Trace.Info($"Creating job server with URL: {jobServerUrl}");
var jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential);
await jobServer.ConnectAsync(jobConnection);
var jobServerQueue = HostContext.GetService<IJobServerQueue>();
jobServerQueue.Start(message);
IExecutionContext jobContext = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService<IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
// Set agent variables.
AgentSettings settings = HostContext.GetService<IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath());
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), "vstsom"));
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService<ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = ReplaceWithConfigUriBase(new Uri(jobContext.Variables.System_TaskDefinitionsUri));
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = ReplaceWithConfigUriBase(new Uri(jobContext.Variables.System_TFCollectionUrl));
}
var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist(jobRequestCancellationToken))
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService<IConfigurationStore>();
taskServerUri = ReplaceWithConfigUriBase(new Uri(configStore.GetSettings().ServerUrl));
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extensions.
Trace.Info("Getting job extensions.");
string hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService<IExtensionManager>();
IJobExtension[] extensions =
(extensionManager.GetExtensions<IJobExtension>() ?? new List<IJobExtension>())
.Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase))
.ToArray();
// Add the prepare steps.
Trace.Info("Adding job prepare extensions.");
List<IStep> steps = new List<IStep>();
foreach (IJobExtension extension in extensions)
{
if (extension.PrepareStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}.");
extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName);
steps.Add(extension.PrepareStep);
}
}
// Add the task steps.
Trace.Info("Adding tasks.");
foreach (TaskInstance taskInstance in message.Tasks)
{
Trace.Verbose($"Adding {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService<ITaskRunner>();
//// TODO: Presently 'TimeoutInMinutes' is not available in DistributedTask.WebApi TaskInstance.
//// Next update it will be available. Once it is available, pass timeout to CreateChild. It will enable task timeout feature.
//// var timeout = taskInstance.TimeoutInMinutes > 0 ? TimeSpan.FromMinutes(taskInstance.TimeoutInMinutes) : Timeout.InfiniteTimeSpan;
taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName);
taskRunner.TaskInstance = taskInstance;
steps.Add(taskRunner);
}
// Add the finally steps.
Trace.Info("Adding job finally extensions.");
foreach (IJobExtension extension in extensions)
{
if (extension.FinallyStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}.");
extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName);
steps.Add(extension.FinallyStep);
}
}
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService<ITaskManager>();
try
{
await taskManager.DownloadAsync(jobContext, message.Tasks);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return jobContext.Complete(TaskResult.Canceled);
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}");
jobContext.Error(ex);
return jobContext.Complete(TaskResult.Failed);
}
// Run the steps.
var stepsRunner = HostContext.GetService<IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, steps);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return jobContext.Complete(TaskResult.Canceled);
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return jobContext.Complete(TaskResult.Failed);
}
Trace.Info($"Job result: {jobContext.Result}");
// Complete the job.
Trace.Info("Completing the job execution context.");
return jobContext.Complete();
}
finally
{
// Drain the job server queue.
if (jobServerQueue != null)
{
try
{
Trace.Info("Shutting down the job server queue.");
await jobServerQueue.ShutdownAsync();
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}");
}
}
}
}
public async Task RunIPCEndToEnd()
{
using (var server = new ProcessChannel())
{
JobRequestMessage result = null;
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = null;
JobEnvironment environment = new JobEnvironment();
List <TaskInstance> tasks = new List <TaskInstance>();
Guid JobId = Guid.NewGuid();
var jobRequest = new JobRequestMessage(plan, timeline, JobId, "someJob", environment, tasks);
Process jobProcess;
server.StartServer((p1, p2) =>
{
string clientFileName = $"Test{IOUtil.ExeExtension}";
jobProcess = new Process();
jobProcess.StartInfo.FileName = clientFileName;
jobProcess.StartInfo.Arguments = "spawnclient " + p1 + " " + p2;
jobProcess.EnableRaisingEvents = true;
jobProcess.Start();
});
var cs = new CancellationTokenSource();
await server.SendAsync(MessageType.NewJobRequest, JsonUtility.ToString(jobRequest), cs.Token);
var packetReceiveTask = server.ReceiveAsync(cs.Token);
Task[] taskToWait = { packetReceiveTask, Task.Delay(5000) };
await Task.WhenAny(taskToWait);
bool timedOut = !packetReceiveTask.IsCompleted;
// Wait until response is received
if (timedOut)
{
cs.Cancel();
try
{
await packetReceiveTask;
}
catch (OperationCanceledException)
{
// Ignore OperationCanceledException and TaskCanceledException exceptions
}
catch (AggregateException errors)
{
// Ignore OperationCanceledException and TaskCanceledException exceptions
errors.Handle(e => e is OperationCanceledException);
}
}
else
{
result = JsonUtility.FromString <JobRequestMessage>(packetReceiveTask.Result.Body);
}
// Wait until response is received
if (timedOut)
{
Assert.True(false, "Test timed out.");
}
else
{
Assert.True(jobRequest.JobId.Equals(result.JobId) && jobRequest.JobName.Equals(result.JobName));
}
}
}
public async Task RunIPCEndToEnd()
{
using (var server = new ProcessChannel())
{
JobRequestMessage result = null;
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = null;
JobEnvironment environment = new JobEnvironment();
List<TaskInstance> tasks = new List<TaskInstance>();
Guid JobId = Guid.NewGuid();
var jobRequest = new JobRequestMessage(plan, timeline, JobId, "someJob", environment, tasks);
Process jobProcess;
server.StartServer((p1, p2) =>
{
string clientFileName = $"Test{IOUtil.ExeExtension}";
jobProcess = new Process();
jobProcess.StartInfo.FileName = clientFileName;
jobProcess.StartInfo.Arguments = "spawnclient " + p1 + " " + p2;
jobProcess.EnableRaisingEvents = true;
jobProcess.Start();
});
var cs = new CancellationTokenSource();
await server.SendAsync(MessageType.NewJobRequest, JsonUtility.ToString(jobRequest), cs.Token);
var packetReceiveTask = server.ReceiveAsync(cs.Token);
Task[] taskToWait = { packetReceiveTask, Task.Delay(30*1000) };
await Task.WhenAny(taskToWait);
bool timedOut = !packetReceiveTask.IsCompleted;
// Wait until response is received
if (timedOut)
{
cs.Cancel();
try
{
await packetReceiveTask;
}
catch (OperationCanceledException)
{
// Ignore OperationCanceledException and TaskCanceledException exceptions
}
catch (AggregateException errors)
{
// Ignore OperationCanceledException and TaskCanceledException exceptions
errors.Handle(e => e is OperationCanceledException);
}
}
else
{
result = JsonUtility.FromString<JobRequestMessage>(packetReceiveTask.Result.Body);
}
// Wait until response is received
if (timedOut)
{
Assert.True(false, "Test timed out.");
}
else
{
Assert.True(jobRequest.JobId.Equals(result.JobId) && jobRequest.JobName.Equals(result.JobName));
}
}
}
