/// <summary>
/// Cleans up TT API objects
/// </summary>
public void Dispose()
{
lock (m_lock)
{
if (!m_disposed)
{
// Unattached callbacks and dispose of all subscriptions
foreach (InstrumentLookupSubscription req in m_lreq)
{
if (req != null)
{
req.Update -= req_Update;
req.Dispose();
}
}
foreach (TimeAndSalesSubscription tsSub in m_ltsSub)
{
if (tsSub != null)
{
tsSub.Update -= tsSub_Update;
tsSub.Dispose();
}
}
// Shutdown the Dispatcher
if (m_disp != null)
{
m_disp.BeginInvokeShutdown();
m_disp = null;
}
m_disposed = true;
}
}
}
/// <summary>
/// Create and start the Dispatcher
/// </summary>
public void Start()
{
// Attach a WorkerDispatcher to the current thread
m_disp = Dispatcher.AttachWorkerDispatcher();
m_disp.BeginInvoke(new Action(Init));
m_disp.Run();
}
public async Task WaitAsync(CancellationToken token)
{
WorkerDispatcher currentDispatch = null;
Guid dispatchedJobId;
if (_jobDispatchedQueue.Count > 0)
{
dispatchedJobId = _jobDispatchedQueue.Dequeue();
if (_jobInfos.TryGetValue(dispatchedJobId, out currentDispatch))
{
Trace.Verbose($"Retrive previous WorkerDispather for job {currentDispatch.JobId}.");
}
}
else
{
Trace.Verbose($"There is no running WorkerDispather needs to await.");
}
if (currentDispatch != null)
{
using (var registration = token.Register(() => { if (currentDispatch.Cancel(TimeSpan.FromSeconds(60)))
{
Trace.Verbose($"Fired cancellation token for job request {currentDispatch.JobId}.");
}
}))
{
try
{
Trace.Info($"Waiting WorkerDispather for job {currentDispatch.JobId} run to finish.");
await currentDispatch.WorkerDispatch;
Trace.Info($"Job request {currentDispatch.JobId} processed successfully.");
}
catch (Exception ex)
{
Trace.Error($"Worker Dispatch failed with an exception for job request {currentDispatch.JobId}.");
Trace.Error(ex);
}
finally
{
WorkerDispatcher workerDispatcher;
if (_jobInfos.TryRemove(currentDispatch.JobId, out workerDispatcher))
{
Trace.Verbose($"Remove WorkerDispather from {nameof(_jobInfos)} dictionary for job {currentDispatch.JobId}.");
workerDispatcher.Dispose();
}
}
}
}
}
public TTDriver(Action <string> onInit, WorkerDispatcher dispatcher)
{
this._dispatcher = dispatcher;
_ttNetApiFunctions = new TTNetApiFunctions(onInit, this._dispatcher);
// Add your app secret Key here. It looks like: 00000000-0000-0000-0000-000000000000:00000000-0000-0000-0000-000000000000
string appSecretKey = "7f6d1dac-4e09-00a4-1a60-b227db734ba9:f29145c6-d8a1-5b88-1a6f-5770ac26211a";
//Set the environment the app needs to run in here
ServiceEnvironment environment = ServiceEnvironment.ProdLive;
this._apiConfig = new TTAPIOptions(
environment,
appSecretKey,
5000);
}
/// <summary>
/// Create and start the Dispatcher
/// </summary>
public void Initialize()
{
if (CheckArchitecture())
{
// Attach a WorkerDispatcher to the current thread
_disp = Dispatcher.AttachWorkerDispatcher();
_disp.BeginInvoke(Init);
_disp.Run();
var delay = DateTime.Now.AddSeconds(30);
while (DateTime.Now.CompareTo(delay) > 0)
{
Dispose();
}
}
}
}//Dispose()
protected virtual void Dispose(bool disposing)
{
bool isOkayToStop = true; // if TTAPI is now requested to shutdown, we have to wait for their callback before we shutdown
lock (m_DisposeLock)
{
if (!m_IsDisposed)
{
if (disposing)
{
if (m_UAPI != null)
{
TTAPI.ShutdownCompleted += new EventHandler(TTAPI_ShutdownCompleted);
TTAPI.Shutdown();
isOkayToStop = false;
}
if (m_XAPI != null)
{
TTAPI.ShutdownCompleted += new EventHandler(TTAPI_ShutdownCompleted);
TTAPI.Shutdown();
isOkayToStop = false;
}
if (isOkayToStop)
{
if (m_Log != null)
{
m_Log.RequestStop();
m_Log = null;
}
if (m_Dispatcher != null)
{
m_Dispatcher.BeginInvokeShutdown();
m_Dispatcher = null;
}
}
}
}
m_IsDisposed = true;
if (isOkayToStop) // if we aren't waiting for TT to shutdown we can call our own shutdown
{
OnStopping();
}
}
}//Dispose()
}//TT_LicenseIssueEventHandler()
//
//
// *********************************************************************
// **** TT API Shut Down Complete ****
// *********************************************************************
/// <summary>
/// TT callback that is has properly shut down the TTAPI instance and we can complete our shutdown
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void TTAPI_ShutdownCompleted(object sender, EventArgs e)
{
m_UAPI = null; // release pointers
m_XAPI = null;
if (m_Log != null)
{
m_Log.RequestStop();
m_Log = null;
}
if (m_Dispatcher != null)
{
m_Dispatcher.BeginInvokeShutdown();
m_Dispatcher = null;
}
OnStopping(); // signal our own shutdown
}
public async Task DispatchToWorker()
{
int counter1 = 0;
var dispatcher1 = new WorkerDispatcher(e => { }, onDispatch: () => counter1++);
int counter2 = 0;
var dispatcher2 = new WorkerDispatcher(e => { }, onDispatch: () => counter2++);
Task.Run(async() => await Task.Delay(100)).ThenOn(dispatcher1, () => counter1.Should().Be(1));
Task.Run(async() => await Task.Delay(200)).ThenOn(dispatcher1, () => counter1.Should().Be(2));
Task.Run(async() => await Task.Delay(100)).ThenOn(dispatcher2, () => counter2.Should().Be(1));
Task.Run(async() => await Task.Delay(200)).ThenOn(dispatcher2, () => counter2.Should().Be(2));
await Task.Delay(500);
counter1.Should().Be(2);
counter2.Should().Be(2);
dispatcher1.Faults.Count().Should().Be(0);
dispatcher2.Faults.Count().Should().Be(0);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary> Event notification for status of authentication. </summary>
////////////////////////////////////////////////////////////////////////////////////////////////////
public void m_api_TTAPIStatusUpdate(object sender, TTAPIStatusUpdateEventArgs e)
{
Console.WriteLine("TTAPIStatusUpdate: {0}", e);
if (e.IsReady == false)
{
// TODO: Do any connection lost processing here
return;
}
// TODO: Do any connection up processing here
// note: can happen multiple times with your application life cycle
WorkerDispatcher dispatcher = new WorkerDispatcher("InstCat");
dispatcher.Run();
InstrumentCatalogSubscription asSpreads = new InstrumentCatalogSubscription(Product.Autospreader, dispatcher);
asSpreads.OnData += OnSpreadDefinitionNotification;
asSpreads.Start();
}
public void Run(JobRequestMessage jobRequestMessage)
{
Trace.Info($"Job request {jobRequestMessage.JobId} received.");
WorkerDispatcher currentDispatch = null;
if (_jobDispatchedQueue.Count > 0)
{
Guid dispatchedJobId = _jobDispatchedQueue.Dequeue();
if (_jobInfos.TryGetValue(dispatchedJobId, out currentDispatch))
{
Trace.Verbose($"Retrive previous WorkerDispather for job {currentDispatch.JobId}.");
}
}
WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token);
_jobInfos.TryAdd(newDispatch.JobId, newDispatch);
_jobDispatchedQueue.Enqueue(newDispatch.JobId);
}
static void Main(string[] args)
{
var dispatcher = new WorkerDispatcher(
onException: e => Console.Error.Write(e),
priority: ThreadPriority.AboveNormal,
threadCount: 10,
faultBufferSize: 2);
for (int i = 0; i < 100; i++)
{
int number = i;
dispatcher.Dispatch(() =>
{
Thread.Sleep(100);
if (number == 81)
{
throw new NotImplementedException("Encountered 81!");
}
Console.WriteLine(number);
});
}
Console.WriteLine("Done dispatching");
Console.ReadLine();
dispatcher.Stop();
foreach (var fault in dispatcher.Faults)
{
Console.WriteLine(fault);
Console.WriteLine("Dispatcher Trace: ");
foreach (var frame in fault.DispatcherStack)
{
Console.WriteLine(frame);
}
Console.WriteLine("Exception Trace: ");
foreach (var frame in fault.ExceptionStack)
{
Console.WriteLine(frame);
}
}
Console.ReadLine();
}
/// <summary>
/// This is called asynchronously by the TTService worker thread. It then creates and
/// attaches a Dispatcher for itself, and calls the InitAPI() method. This creates a
/// queue for the TTService worker thread, which keeps it alive after processing requests.
/// </summary>
private void StartInitAPI()
{
if (m_Dispatcher == null)
{
m_Dispatcher = Dispatcher.AttachWorkerDispatcher();
}
m_Dispatcher.BeginInvoke(new Action(InitAPI));
try
{ // this is a blocking statement, the thread will return here whenever a call is returned.
m_Dispatcher.Run(); // This keeps the thread alive at the end of a return.
}
catch (Exception e)
{
if (m_Log != null)
{
m_Log.NewEntry(LogLevel.Error, "StartInitAPI: Error. Failed to start TT API. {0}", e.Message);
}
Thread.Sleep(2000); // worker should sleep, then try to restart.
StartInitAPI(); // try again.
}
}// StartInitAPI().
/// <summary>
/// Shuts down the TT API
/// </summary>
public void Dispose()
{
lock (m_lock)
{
if (!disposed)
{
// Shutdown the TT API
if (apiInstance != null)
{
apiInstance.Shutdown();
apiInstance = null;
}
// Shutdown the Dispatcher
if (workDispatch != null)
{
workDispatch.BeginInvokeShutdown();
workDispatch = null;
}
disposed = true;
}
}
}
public void Run(JobRequestMessage jobRequestMessage)
{
Trace.Info($"Job request {jobRequestMessage.JobId} received.");
WorkerDispatcher currentDispatch = null;
if (_jobDispatchedQueue.Count > 0)
{
Guid dispatchedJobId = _jobDispatchedQueue.Dequeue();
if (_jobInfos.TryGetValue(dispatchedJobId, out currentDispatch))
{
Trace.Verbose($"Retrive previous WorkerDispather for job {currentDispatch.JobId}.");
}
}
WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
newDispatch.WorkerDispatch = RunAsync(jobRequestMessage, currentDispatch, newDispatch.WorkerCancellationTokenSource.Token);
_jobInfos.TryAdd(newDispatch.JobId, newDispatch);
_jobDispatchedQueue.Enqueue(newDispatch.JobId);
}
protected virtual void Dispose(bool disposing)
{
if (!disposed)
{
if (disposing)
{
// Shutdown all subscriptions
if (req != null)
{
req.Dispose();
req = null;
}
if (ps != null)
{
ps.Dispose();
ps = null;
}
// Shutdown the Dispatcher
if (disp != null)
{
disp.BeginInvokeShutdown();
disp = null;
}
// Shutdown the TT API
if (apiInstance != null)
{
//TTAPI.ShutdownCompleted += new EventHandler();
TTAPI.Shutdown();
apiInstance = null;
}
}
}
disposed = true;
}
public LiveFuturesSubscriber(KafkaClient awsKafkaClient, KafkaClient njKafkaClient, WorkerDispatcher dispatcher)
{
_awsKafkaClient = awsKafkaClient;
_njKafkaClient = njKafkaClient;
_dispatcher = dispatcher;
}
public HttpService(HttpListener listener, StreamSocketAdapter streamSocketAdapter, WorkerDispatcher workerDispatcher)
{
_listener = listener;
_streamSocketAdapter = streamSocketAdapter;
_workerDispatcher = workerDispatcher;
}
private async Task RunAsync(JobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = message.LockToken;
// start renew job request
Trace.Info("Start renew job request.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
// TODO: not need to return anything.
Trace.Info("Unable to renew job request for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
await CompleteJobRequestAsync(_poolId, requestId, lockToken, TaskResult.Canceled);
return;
}
Task <int> workerProcessTask = null;
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
// Start the child process.
var assemblyDirectory = IOUtil.GetBinPath();
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info("Send job request message to worker.");
using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 45 sec. kill worker.
Trace.Info("Job request message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info("Worker finished. Code: " + returnCode);
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result));
// complete job request
await CompleteJobRequestAsync(_poolId, requestId, lockToken, result);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after 45 sec
try
{
Trace.Info("Send job cancellation message to worker.");
using (var csSendCancel = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.CancelRequest,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info("Job cancel message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
// wait worker to exit within 45 sec, then kill worker.
using (var csKillWorker = new CancellationTokenSource(TimeSpan.FromSeconds(45)))
{
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, csKillWorker.Token));
}
// worker haven't exit within 45 sec.
if (completedTask != workerProcessTask)
{
Trace.Info("worker process haven't exit after 45 sec, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
Trace.Info($"finish job request with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
//TODO: don't finish job request on abandon
await CompleteJobRequestAsync(_poolId, requestId, lockToken, resultOnAbandonOrCancel);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
}
}
}
protected virtual void Dispose(bool disposing)
{
if (!disposed)
{
if (disposing)
{
// Shutdown all subscriptions
if (req1 != null)
{
req1.Dispose();
req1 = null;
}
if (req2 != null)
{
req2.Dispose();
req2 = null;
}
if (casReq != null)
{
casReq.Dispose();
casReq = null;
}
if (ts != null)
{
ts.Dispose();
ts = null;
}
// Shutdown the Dispatcher
if (disp != null)
{
disp.BeginInvokeShutdown();
disp = null;
}
// Shutdown the TT API
if (apiInstance != null)
{
apiInstance.Shutdown();
apiInstance = null;
}
}
}
disposed = true;
}
public void Init()
{
dispatcher = new WorkerDispatcher();
dispatcher.Start(10);
flag = false;
}
/// <summary>
/// Event notification for completion of TT API shutdown
/// </summary>
public void TTAPI_ShutdownCompleted(object sender, EventArgs e)
{
// Shutdown the Dispatcher
if (m_disp != null)
{
m_disp.BeginInvokeShutdown();
m_disp = null;
}
// Dispose of any other objects / resources
}
private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobDisplayName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat
// start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request with result Cancelled
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
HostContext.WritePerfCounter($"JobRequestRenewed_{requestId.ToString()}");
Task <int> workerProcessTask = null;
object _outputLock = new object();
List <string> workerOutput = new List <string>();
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
if (HostContext.RunMode == RunMode.Normal)
{
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (_outputLock)
{
workerOutput.Add(stdout.Data);
}
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (_outputLock)
{
workerOutput.Add(stderr.Data);
}
}
};
}
else if (HostContext.RunMode == RunMode.Local)
{
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
}
// Start the child process.
HostContext.WritePerfCounter("StartingWorkerProcess");
var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin);
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: true,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info($"Send job request message to worker for job {message.JobId}.");
HostContext.WritePerfCounter($"AgentSendingJobToWorker_{message.JobId}");
using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 30 sec. kill worker.
Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
// we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner.
await notification.JobStarted(message.JobId);
HostContext.WritePerfCounter($"SentJobToWorker_{requestId.ToString()}");
try
{
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);
string detailInfo = null;
if (!TaskResultUtil.IsValidReturnCode(returnCode))
{
detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
await LogWorkerProcessUnhandledException(message, detailInfo);
}
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request for job {message.JobId} with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, result));
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
// print out unhandled exception happened in worker after we complete job request.
// when we run out of disk space, report back to server has higher priority.
if (!string.IsNullOrEmpty(detailInfo))
{
Trace.Error("Unhandled exception happened in worker:");
Trace.Error(detailInfo);
}
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after worker cancel timeout
try
{
Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
{
var messageType = MessageType.CancelRequest;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
messageType = MessageType.AgentShutdown;
break;
case ShutdownReason.OperatingSystemShutdown:
messageType = MessageType.OperatingSystemShutdown;
break;
}
}
await processChannel.SendAsync(
messageType : messageType,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
// wait worker to exit
// if worker doesn't exit within timeout, then kill worker.
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
// worker haven't exit within cancellation timeout.
if (completedTask != workerProcessTask)
{
Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
}
finally
{
// This should be the last thing to run so we don't notify external parties until actually finished
await notification.JobCompleted(message.JobId);
}
}
}
}
private async Task EnsureDispatchFinished(WorkerDispatcher jobDispatch, bool cancelRunningJob = false)
{
if (!jobDispatch.WorkerDispatch.IsCompleted)
{
if (cancelRunningJob)
{
// cancel running job when shutting down the agent.
// this will happen when agent get Ctrl+C or message queue loop crashed.
jobDispatch.WorkerCancellationTokenSource.Cancel();
// wait for worker process exit then return.
await jobDispatch.WorkerDispatch;
return;
}
// base on the current design, server will only send one job for a given agent everytime.
// if the agent received a new job request while a previous job request is still running, this typically indicate two situations
// 1. an agent bug cause server and agent mismatch on the state of the job request, ex. agent not renew jobrequest properly but think it still own the job reqest, however server already abandon the jobrequest.
// 2. a server bug or design change that allow server send more than one job request to an given agent that haven't finish previous job request.
var agentServer = HostContext.GetService <IAgentServer>();
TaskAgentJobRequest request = null;
try
{
request = await agentServer.GetAgentRequestAsync(_poolId, jobDispatch.RequestId, CancellationToken.None);
}
catch (Exception ex)
{
// we can't even query for the jobrequest from server, something totally busted, stop agent/worker.
Trace.Error($"Catch exception while checking jobrequest {jobDispatch.JobId} status. Cancel running worker right away.");
Trace.Error(ex);
jobDispatch.WorkerCancellationTokenSource.Cancel();
// make sure worker process exit before we rethrow, otherwise we might leave orphan worker process behind.
await jobDispatch.WorkerDispatch;
// rethrow original exception
throw;
}
if (request.Result != null)
{
// job request has been finished, the server already has result.
// this means agent is busted since it still running that request.
// cancel the zombie worker, run next job request.
Trace.Error($"Received job request while previous job {jobDispatch.JobId} still running on worker. Cancel the previous job since the job request have been finished on server side with result: {request.Result.Value}.");
jobDispatch.WorkerCancellationTokenSource.Cancel();
// wait 45 sec for worker to finish.
Task completedTask = await Task.WhenAny(jobDispatch.WorkerDispatch, Task.Delay(TimeSpan.FromSeconds(45)));
if (completedTask != jobDispatch.WorkerDispatch)
{
// at this point, the job exectuion might encounter some dead lock and even not able to be canclled.
// no need to localize the exception string should never happen.
throw new InvalidOperationException($"Job dispatch process for {jobDispatch.JobId} has encountered unexpected error, the dispatch task is not able to be canceled within 45 seconds.");
}
}
else
{
// something seriously wrong on server side. stop agent from continue running.
// no need to localize the exception string should never happen.
throw new InvalidOperationException($"Server send a new job request while the previous job request {jobDispatch.JobId} haven't finished.");
}
}
try
{
await jobDispatch.WorkerDispatch;
Trace.Info($"Job request {jobDispatch.JobId} processed succeed.");
}
catch (Exception ex)
{
Trace.Error($"Worker Dispatch failed witn an exception for job request {jobDispatch.JobId}.");
Trace.Error(ex);
}
finally
{
WorkerDispatcher workerDispatcher;
if (_jobInfos.TryRemove(jobDispatch.JobId, out workerDispatcher))
{
Trace.Verbose($"Remove WorkerDispather from {nameof(_jobInfos)} dictionary for job {jobDispatch.JobId}.");
workerDispatcher.Dispose();
}
}
}
private async Task EnsureDispatchFinished(WorkerDispatcher jobDispatch)
{
if (!jobDispatch.WorkerDispatch.IsCompleted)
{
// base on the current design, server will only send one job for a given agent everytime.
// if the agent received a new job request while a previous job request is still running, this typically indicate two situations
// 1. an agent bug cause server and agent mismatch on the state of the job request, ex. agent not renew jobrequest properly but think it still own the job reqest, however server already abandon the jobrequest.
// 2. a server bug or design change that allow server send more than one job request to an given agent that haven't finish previous job request.
var agentServer = HostContext.GetService<IAgentServer>();
TaskAgentJobRequest request = await agentServer.GetAgentRequestAsync(_poolId, jobDispatch.RequestId, CancellationToken.None);
if (request.Result != null)
{
// job request has been finished, the server already has result.
// this means agent is busted since it still running that request.
// cancel the zombie worker, run next job request.
Trace.Error($"Received job request while previous job {jobDispatch.JobId} still running on worker. Cancel the previous job since the job request have been finished on server side with result: {request.Result.Value}.");
jobDispatch.WorkerCancellationTokenSource.Cancel();
// wait 45 sec for worker to finish.
Task completedTask = await Task.WhenAny(jobDispatch.WorkerDispatch, Task.Delay(TimeSpan.FromSeconds(45)));
if (completedTask != jobDispatch.WorkerDispatch)
{
// at this point, the job exectuion might encounter some dead lock and even not able to be canclled.
// no need to localize the exception string should never happen.
throw new InvalidOperationException("Job dispatch process has encountered unexpected error, the dispatch task is not able to be canceled within 45 seconds.");
}
}
else
{
// something seriously wrong on server side. stop agent from continue running.
// no need to localize the exception string should never happen.
throw new InvalidOperationException("Server send a new job request while the previous job request haven't finished.");
}
}
try
{
await jobDispatch.WorkerDispatch;
Trace.Info($"Job request {jobDispatch.JobId} processed succeed.");
}
catch (Exception ex)
{
Trace.Error($"Worker Dispatch failed witn an exception for job request {jobDispatch.JobId}.");
Trace.Error(ex);
}
finally
{
WorkerDispatcher workerDispatcher;
if (_jobInfos.TryRemove(jobDispatch.JobId, out workerDispatcher))
{
Trace.Verbose($"Remove WorkerDispather from {nameof(_jobInfos)} dictionary for job {jobDispatch.JobId}.");
workerDispatcher.Dispose();
}
}
}
//
//
#endregion//Public Methods
#region Private Methods
// *****************************************************************
// **** Private Methods ****
// *****************************************************************
/// <summary>
/// This is the first method called by the FillListener's worker thread.
/// To capture itself permanantly, it will call its own dispatcher and complete its
/// initialization.
/// </summary>
private void InitializeThread()
{
m_Dispatcher = TradingTechnologies.TTAPI.Dispatcher.AttachWorkerDispatcher();
m_Dispatcher.BeginInvoke(new Action(InitComplete));
m_Dispatcher.Run(); // this tells thread to do this and wait.
}
protected virtual void Dispose(bool disposing)
{
if (!disposed)
{
if (disposing)
{
// Shutdown all subscriptions
if (fs != null)
{
fs.Dispose();
fs = null;
}
// Shutdown the Dispatcher
if (disp != null)
{
disp.BeginInvokeShutdown();
disp = null;
}
// Shutdown the TT API
if (apiInstance != null)
{
apiInstance.Shutdown();
apiInstance = null;
}
}
}
disposed = true;
}
private async Task RunAsync(JobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService<ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName));
// first job request renew succeed.
TaskCompletionSource<int> firstJobRequestRenewed = new TaskCompletionSource<int>();
var notification = HostContext.GetService<IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
await notification.JobStarted(message.JobId);
long requestId = message.RequestId;
Guid lockToken = message.LockToken;
// start renew job request
Trace.Info("Start renew job request.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
// TODO: not need to return anything.
Trace.Info("Unable to renew job request for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
Task<int> workerProcessTask = null;
using (var processChannel = HostContext.CreateService<IProcessChannel>())
using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
// Start the child process.
var assemblyDirectory = IOUtil.GetBinPath();
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info("Send job request message to worker.");
using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType: MessageType.NewJobRequest,
body: JsonUtility.ToString(message),
cancellationToken: csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 45 sec. kill worker.
Trace.Info("Job request message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info("Worker finished. Code: " + returnCode);
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result));
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after 45 sec
try
{
Trace.Info("Send job cancellation message to worker.");
using (var csSendCancel = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType: MessageType.CancelRequest,
body: string.Empty,
cancellationToken: csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info("Job cancel message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
// wait worker to exit within 45 sec, then kill worker.
using (var csKillWorker = new CancellationTokenSource(TimeSpan.FromSeconds(45)))
{
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, csKillWorker.Token));
}
// worker haven't exit within 45 sec.
if (completedTask != workerProcessTask)
{
Trace.Info("worker process haven't exit after 45 sec, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
Trace.Info($"finish job request with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
//TODO: don't finish job request on abandon
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
}
}
}
public TTSettlesSubscriber(KafkaClient awsKafkaClient, KafkaClient njKafkaClient, WorkerDispatcher dispatcher, InstrumentType instrumentType)
{
this._awsKafkaClient = awsKafkaClient;
this._njKafkaClient = njKafkaClient;
this._dispatcher = dispatcher;
this._instrumentType = instrumentType;
}
public TTNetApiFunctions(Action <string> onInitialised, WorkerDispatcher dispatcher)
{
this.onInit = onInitialised;
m_disp = dispatcher;
}