/// <summary>
/// This method sets the task for execution and calls the end method on completion.
/// </summary>
/// <param name="tracker">The tracker to enqueue.</param>
private void ExecuteTask(TaskTracker tracker)
{
TaskTrackerEvent(DiagnosticOnExecuteTaskBefore, tracker);
try
{
var tOpts = tracker.IsLongRunning ? TaskCreationOptions.LongRunning : TaskCreationOptions.None;
if (mTaskRequests.TryAdd(tracker.Id, tracker))
{
mAvailability.Increment(tracker);
tracker.UTCExecute = DateTime.UtcNow;
try
{
var task = Task.Factory.StartNew
(
async() => await ExecuteTaskCreate(tracker), tracker.Cts.Token, tOpts, TaskScheduler.Default
)
.Unwrap();
task.ContinueWith((t) => ExecuteTaskComplete(tracker, task.IsCanceled || task.IsFaulted, t.Exception));
}
catch (Exception ex)
{
ExecuteTaskComplete(tracker, true, ex);
Collector?.LogException("Task creation failed.", ex);
}
return;
}
else
{
TaskTrackerEvent(DiagnosticOnExecuteTaskBeforeEnqueueFailed, tracker);
Collector?.LogMessage(LoggingLevel.Error, $"Task could not be enqueued: {tracker.Id}-{tracker.Caller}");
}
}
catch (Exception ex)
{
tracker.IsFailure = true;
tracker.FailureException = ex;
TaskTrackerEvent(DiagnosticOnExecuteTaskBeforeException, tracker);
Collector?.LogException("ExecuteTask execute exception", ex);
}
try
{
tracker.Cancel();
}
catch (Exception ex)
{
Collector?.LogException("ExecuteTask unhandled exception.", ex);
}
}
/// <summary>
/// This method takes incoming messages from the commands. It is the method set on the Dispatcher property.
/// </summary>
/// <param name="service">The calling command.</param>
/// <param name="id">The callback id. If this is not set then timeouts will not be signalled.</param>
/// <param name="payload">The payload to process.</param>
public virtual void ExecuteOrEnqueue(ICommand service, string id, TransmissionPayload payload)
{
TaskTracker tracker = TrackerCreateFromPayload(payload, service.GetType().Name);
if (id != null)
{
tracker.CallbackId = id;
tracker.Callback = service;
}
ExecuteOrEnqueue(tracker);
}
/// <summary>
/// Initializes a new instance of the <see cref="CommandHarnessTraffic"/> class.
/// </summary>
/// <param name="direction">The request direction: incoming or outgoing.</param>
/// <param name="tracker">The request.</param>
/// <param name="referenceId">The reference identifier.</param>
/// <param name="originatorId">Initial identifier.</param>
public CommandHarnessTraffic(CommandHarnessTrafficDirection direction
, TaskTracker tracker
, string referenceId
, long?originatorId = null)
{
Direction = direction;
Id = tracker?.Id ?? Guid.NewGuid();
Tracker = tracker;
Responses = new List <TransmissionPayload>();
ReferenceId = referenceId;
OriginatorId = originatorId;
}
public bool TryDequeue(out TaskTracker item)
{
item = null;
KeyValuePair <TaskTracker, int> wrapper;
if (mQueue.TryDequeue(out wrapper))
{
item = wrapper.Key;
StatisticsInternal.ActiveDecrement(wrapper.Value);
return(true);
}
return(false);
}
/// <summary>
/// This method attemps to dequeue a message.
/// </summary>
/// <param name="item">The item dequeued or null</param>
/// <returns>Returns true if a message was dequeued successfully.</returns>
public bool TryDequeue(out TaskTracker item)
{
item = null;
QueueTrackerHolder wrapper;
if (mQueue.TryDequeue(out wrapper))
{
item = wrapper.Item;
StatisticsInternal.ActiveDecrement(wrapper.Ingress);
return(true);
}
return(false);
}
/// <summary>
/// This method processes an individual payload returned from a client.
/// </summary>
/// <param name="clientId">The originating client.</param>
/// <param name="payload">The payload.</param>
private void PayloadSubmit(Guid clientId, TransmissionPayload payload)
{
try
{
if (payload.Message.ChannelPriority < 0)
{
payload.Message.ChannelPriority = 0;
}
mClientCollection.QueueTimeLog(clientId, payload.Message.EnqueuedTimeUTC);
mClientCollection.ActiveIncrement(clientId);
//Verify the incoming payload with the security container.
PayloadIncomingSecurity(payload);
//Do we need to redirect the payload based on the redirect/rewrite rules.
PayloadIncomingRedirectCheck(payload);
TaskTracker tracker = TaskManager.TrackerCreateFromPayload(payload, payload.Source);
tracker.ExecuteComplete = (tr, failed, ex) =>
{
try
{
var contextPayload = tr.Context as TransmissionPayload;
mClientCollection.ActiveDecrement(clientId, tr.TickCount);
if (failed)
{
mClientCollection.ErrorIncrement(clientId);
}
contextPayload.Signal(!failed);
}
catch (Exception exin)
{
Collector?.LogException($"Payload completion error-{payload} after {(tr.Context as TransmissionPayload)?.Message?.FabricDeliveryCount} delivery attempts", exin);
}
};
//Submit the tracker to the task manager.
TaskSubmit(tracker);
}
catch (Exception ex)
{
Collector?.LogException($"ProcessClientPayload: unhandled error {payload.Source}/{payload.Message.CorrelationKey}-{payload} after {payload.Message?.FabricDeliveryCount} delivery attempts", ex);
payload.SignalFail();
}
}
/// <summary>
/// This method creates the necessary task.
/// </summary>
/// <param name="tracker">The tracker to create a task.</param>
/// <returns>Returns a task for the job.</returns>
private Task ExecuteTaskCreate(TaskTracker tracker)
{
//Internal tasks should not block other incoming tasks and they are pass-through requests from another task.
tracker.ExecuteTickCount = StatisticsInternal.ActiveIncrement();
var payload = tracker.Context as TransmissionPayload;
if (payload != null)
{
payload.Cancel = tracker.Cts.Token;
tracker.ExecuteTask = Dispatcher(payload);
}
else
{
tracker.ExecuteTask = tracker.Execute(tracker.Cts.Token);
}
return(tracker.ExecuteTask);
}
/// <summary>
/// This method cancels the specific tracker.
/// </summary>
/// <param name="tracker">The tracker to cancel.</param>
private void TaskCancel(TaskTracker tracker)
{
if (tracker.IsCancelled)
{
return;
}
try
{
StatisticsInternal.TimeoutRegister(1);
tracker.Cancel();
LoopSet();
}
catch (Exception ex)
{
Collector?.LogException("TaskCancel exception", ex);
}
TaskTrackerEvent(DiagnosticOnExecuteTaskCancelled, tracker);
}
/// <summary>
/// This private method builds the payload consistently for the incoming payload.
/// </summary>
/// <param name="schedule">The schedule to add to a tracker.</param>
/// <returns>Returns a tracker of type payload.</returns>
private TaskTracker TaskTrackerCreate(Schedule schedule)
{
TaskTracker tracker = new TaskTracker(TaskTrackerType.Schedule, null);
tracker.IsLongRunning = schedule.IsLongRunning;
if (schedule.IsInternal)
{
tracker.Priority = TaskTracker.PriorityInternal;
}
else
{
tracker.Priority = 2;
}
tracker.Context = schedule;
tracker.Name = schedule.Name;
return(tracker);
}
public void Enqueue(TaskTracker tracker)
{
var payload = tracker.Context as TransmissionPayload;
int priority = tracker.Priority ?? mLevels - 1;
if (payload != null)
{
priority = payload.Message.ChannelPriority;
}
if (priority > mLevels - 1)
{
priority = mLevels - 1;
}
tracker.Priority = priority;
mTasksQueue[priority].Enqueue(tracker);
}
/// <summary>
/// This method is used to execute a tracker on enqueue until there are task slots available.
/// </summary>
/// <param name="tracker">The tracker.</param>
public void ExecuteOrEnqueue(TaskTracker tracker)
{
//This is where the security validation should be added.
if (tracker.IsInternal)
{
if (mPolicy.ExecuteInternalDirect)
{
ExecuteTask(tracker);
}
else
{
mProcessInternalQueue.Enqueue(tracker);
}
}
else
{
mTasksQueue.Enqueue(tracker);
}
LoopSet();
}
/// <summary>
/// This method checks whether the process is overloaded and schedules a long running task to reduce the overload.
/// </summary>
public virtual void Process()
{
if (!Overloaded)
{
return;
}
TaskTracker tracker = new TaskTracker(TaskTrackerType.Overload, null);
tracker.Name = GetType().Name;
tracker.Caller = GetType().Name;
tracker.IsLongRunning = true;
tracker.Priority = 3;
tracker.Execute = async(token) => await OverloadProcess();
tracker.ExecuteComplete = (t, s, ex) => Interlocked.Decrement(ref mOverloadTaskCount);
Interlocked.Increment(ref mOverloadTaskCount);
TaskSubmit(tracker);
}
/// <summary>
/// This process marks a process a killed.
/// </summary>
/// <param name="tracker">The tracker itself.</param>
private void TaskKill(TaskTracker tracker)
{
if (tracker.IsKilled)
{
return;
}
lock (mSyncKill)
{
if (tracker.IsKilled)
{
return;
}
tracker.IsKilled = true;
mAvailability.Decrement(tracker, true);
LoopSet();
}
TaskTrackerEvent(DiagnosticOnExecuteTaskKilled, tracker);
}
/// <summary>
/// This method builds the task tracker for the listener poll.
/// </summary>
/// <param name="context">The client priority holder context.</param>
/// <returns>Returns a tracker of type listener poll.</returns>
private void TrackerSubmitFromClientPriorityHolder(ClientPriorityHolder context)
{
//Create the task that will poll the client for incoming messages.
TaskTracker tracker = new TaskTracker(TaskTrackerType.ListenerClientPoll, mPolicy.ListenerRequestTimespan)
{
Priority = TaskTracker.PriorityInternal,
Context = context,
Name = context.Name
};
//Set the execute function that polls the context and retrieves incoming payloads from the fabric
//and submits them to be processed.
tracker.Execute = async t =>
{
var currentContext = ((ClientPriorityHolder)tracker.Context);
var payloads = await currentContext.Poll();
if (payloads != null && payloads.Count > 0)
{
foreach (var payload in payloads)
{
PayloadSubmit(currentContext.ClientId, payload);
}
}
};
//Set the completion function that releases the slot reservations back in to the pool.
tracker.ExecuteComplete = (tr, failed, ex) =>
{
var currentContext = ((ClientPriorityHolder)tr.Context);
TaskAvailability.ReservationRelease(currentContext.Id);
currentContext.Release(failed);
};
//Submit the tracker for processing.
TaskSubmit(tracker);
}
/// <summary>
/// This method is used to manually process the schedule.
/// </summary>
/// <param name="tracker">The tracker.</param>
/// <exception cref="System.ArgumentOutOfRangeException">tracker - The tracker object was not of type schedule.</exception>
protected virtual void ProcessScheduleTask(TaskTracker tracker)
{
if (tracker.Type != TaskTrackerType.Schedule)
{
throw new ArgumentOutOfRangeException("tracker", "The tracker object was not of type schedule.");
}
Exception failedEx = null;
try
{
var token = new CancellationToken();
tracker.Execute(token).Wait(token);
}
catch (Exception ex)
{
failedEx = ex;
}
finally
{
tracker.ExecuteComplete(tracker, failedEx != null, failedEx);
}
}
/// <summary>
/// This method adds a tracker to the availability counters.
/// </summary>
/// <param name="tracker">The tracker to add.</param>
/// <returns>Returns the tracker id.</returns>
public long Increment(TaskTracker tracker)
{
if (tracker.ProcessSlot.HasValue && mActiveBag.ContainsKey(tracker.ProcessSlot.Value))
{
throw new ArgumentOutOfRangeException($"The tracker has already been submitted.");
}
tracker.ProcessSlot = Interlocked.Increment(ref mProcessSlot);
if (!mActiveBag.TryAdd(tracker.ProcessSlot.Value, tracker.Id))
{
throw new ArgumentOutOfRangeException($"The tracker has already been submitted.");
}
if (tracker.IsInternal)
{
mPriorityInternal.Increment();
}
else
{
mPriorityStatus[tracker.Priority.Value].Increment();
}
return(tracker.ProcessSlot.Value);
}
public void Enqueue(TaskTracker item)
{
var wrapper = new KeyValuePair <TaskTracker, int>(item, StatisticsInternal.ActiveIncrement());
mQueue.Enqueue(wrapper);
}
/// <summary>
/// This method is used to complete a schedule request and to
/// recalculate the next schedule.
/// </summary>
/// <param name="tracker">The tracker object.</param>
private void ScheduleComplete(TaskTracker tracker, bool failed, Exception ex)
{
var schedule = tracker.Context as Schedule;
schedule.Complete(!failed, isException: failed, lastEx: ex, exceptionTime: DateTime.UtcNow);
}
/// <summary>
/// Identifies whether the tracker context is a payload.
/// </summary>
/// <param name="tracker">The tracker.</param>
/// <returns>Return true if the context is a payload.</returns>
public static bool IsTransmissionPayload(this TaskTracker tracker)
{
return(tracker.Context is TransmissionPayload);
}
/// <summary>
/// This method is called after a task has completed.
/// </summary>
/// <param name="tracker">The tracker.</param>
/// <param name="failed">A boolean value indicating whether the task has failed.</param>
/// <param name="tex">Any exception that was caught as the task executed.</param>
private void ExecuteTaskComplete(TaskTracker tracker, bool failed, Exception tex)
{
tracker.IsFailure = failed;
tracker.FailureException = tex;
tracker.ToTransmissionPayload()?.TraceWrite("ExecuteTaskComplete", "TaskManager");
try
{
TaskTracker outTracker;
if (mTaskRequests.TryRemove(tracker.Id, out outTracker))
{
mAvailability.Decrement(outTracker);
try
{
outTracker.ExecuteComplete?.Invoke(outTracker, failed, tex);
}
catch (Exception ex)
{
//We shouldn't throw an exception here, but let's check just in case.
Collector?.LogException("ExecuteTaskComplete/ExecuteComplete", ex);
}
if (outTracker.ExecuteTickCount.HasValue)
{
StatisticsInternal.ActiveDecrement(outTracker.ExecuteTickCount.Value);
}
}
else
{
tracker.IsFailure = true;
TaskTrackerEvent(DiagnosticOnExecuteTaskCompleteOrphan, tracker);
}
}
catch (Exception ex)
{
Collector?.LogException($"Task {tracker.Id} has faulted when completing: {ex.Message}", ex);
}
try
{
//Signal the poll loop to proceed in case it is waiting, this will check for any pending tasks that require processing.
LoopSet();
if (failed)
{
TaskTrackerEvent(DiagnosticOnExecuteTaskCompleteFailure, tracker);
StatisticsInternal.ErrorIncrement();
if (tex != null && tex is AggregateException)
{
foreach (Exception ex in ((AggregateException)tex).InnerExceptions)
{
Collector?.LogException(string.Format("Task exception {0}-{1}", tracker.Id, tracker.Caller), ex);
}
}
}
else
{
TaskTrackerEvent(DiagnosticOnExecuteTaskCompleteSuccess, tracker);
}
}
catch { }
}
/// <summary>
/// This method takes incoming messages from the initiators.
/// </summary>
/// <param name="payload">The payload to process.</param>
/// <param name="callerName">This is the name of the calling party. It is primarily used for debug and trace reasons.</param>
public virtual void ExecuteOrEnqueue(TransmissionPayload payload, string callerName)
{
TaskTracker tracker = TrackerCreateFromPayload(payload, callerName);
ExecuteOrEnqueue(tracker);
}
/// <summary>
/// Transmissions the payload trace set.
/// </summary>
/// <param name="tracker">The tracker.</param>
/// <param name="eventArgs">The <see cref="TransmissionPayloadTraceEventArgs"/> instance containing the event data.</param>
public static void TransmissionPayloadTraceWrite(this TaskTracker tracker, TransmissionPayloadTraceEventArgs eventArgs)
{
tracker.ToTransmissionPayload()?.TraceWrite(eventArgs);
}
/// <summary>
/// Converts the context to a schedule.
/// </summary>
/// <param name="tracker">The tracker.</param>
/// <returns>Returns the schedule or null.</returns>
public static Schedule ToSchedule(this TaskTracker tracker)
{
return(tracker.Context as Schedule);
}
/// <summary>
/// Converts the context to the transmission payload.
/// </summary>
/// <param name="tracker">The tracker.</param>
/// <returns>Returns the payload or null.</returns>
public static TransmissionPayload ToTransmissionPayload(this TaskTracker tracker)
{
return(tracker.Context as TransmissionPayload);
}
/// <summary>
/// Identifies whether the tracker context is a schedule.
/// </summary>
/// <param name="tracker">The tracker.</param>
/// <returns>Return true if the context is a schedule.</returns>
public static bool HasSchedule(this TaskTracker tracker)
{
return(tracker.Context is Schedule);
}
/// <summary>
/// Transmissions the payload trace set.
/// </summary>
/// <param name="tracker">The tracker.</param>
/// <param name="message">The message.</param>
/// <param name="source">The optional source parameter.</param>
public static void TransmissionPayloadTraceWrite(this TaskTracker tracker, string message, string source = null)
{
tracker.ToTransmissionPayload()?.TraceWrite(message, source);
}
/// <summary>
/// This method processes an individual incoming payload submitted from a client.
/// </summary>
/// <param name="clientId">The originating client.</param>
/// <param name="payload">The payload.</param>
private void PayloadSubmit(Guid clientId, TransmissionPayload payload)
{
try
{
payload.TraceConfigure(mPolicy.TransmissionPayloadTraceEnabled);
payload.TraceWrite("Incoming", "CommunicationContainer/PayloadSubmit");
//Ensure the priority cannot spoof the internal priority of -1
if (payload.Message.ChannelPriority < 0)
{
payload.Message.ChannelPriority = 0;
}
mClientCollection.QueueTimeLog(clientId, payload.Message.EnqueuedTimeUTC);
//Verify the incoming payload with the security container.
PayloadIncomingSecurity(payload);
//Do we need to redirect the payload based on the redirect/rewrite rules.
PayloadIncomingRedirectCheck(payload);
//Create the tracker to process the incoming
TaskTracker tracker = TaskManager.TrackerCreateFromPayload(payload, payload.Source);
//Set the function that executes when the payload completes.
tracker.ExecuteComplete = (tr, failed, ex) =>
{
var contextPayload = tr.ToTransmissionPayload();
try
{
mClientCollection.ActiveDecrement(clientId, tr.TickCount);
if (failed)
{
mClientCollection.ErrorIncrement(clientId);
}
contextPayload.Signal(!failed);
contextPayload.TraceWrite(failed?"Failed":"Success", "CommunicationContainer/PayloadSubmit -> ExecuteComplete");
}
catch (Exception exin)
{
Collector?.LogException($"Payload completion error-{payload} after {(tr.Context as TransmissionPayload)?.Message?.FabricDeliveryCount} delivery attempts", exin);
contextPayload.TraceWrite($"Exception: {ex.Message}", "CommunicationContainer/PayloadSubmit -> ExecuteComplete");
}
};
//Submit the tracker to the task manager.
payload.TraceWrite("Outgoing", "CommunicationContainer/PayloadSubmit");
//Submit the task to be processed.
TaskSubmit(tracker);
}
catch (Exception ex)
{
Collector?.LogException($"ProcessClientPayload: unhandled error {payload.Source}/{payload.Message.CorrelationKey}-{payload} after {payload.Message?.FabricDeliveryCount} delivery attempts", ex);
payload.TraceWrite($"Exception: {ex.Message}", "CommunicationContainer/PayloadSubmit");
payload.SignalFail();
}
}
