public void Started(RFWorkQueueItem i)
{
if (Filter(i))
{
Update(i.Item, DispatchState.Started, null, null);
}
}
private void ProcessInstructionThread(RFWorkQueueItem i)
{
try
{
// To make transaction work we'd have to use a single SQL connection to do all reads and writes
/* using (var ts = new TransactionScope(TransactionScopeOption.Required, new TransactionOptions
* {
* IsolationLevel = System.Transactions.IsolationLevel.ReadCommitted,
* Timeout = TimeSpan.FromSeconds(30)
* }))*/
{
Log.Debug(this, "Started thread to process instruction {0}", i.Item as RFProcessInstruction);
var sink = new RFBufferingSink(); // create thread-local manager to buffer events and instructions, then send them back in bulk
RFProcessingResult result = null;
try
{
result = _context.Engine.Process(i.Item as RFInstruction, _context.GetProcessingContext(i.ProcessingKey, sink, sink, null));
}
catch (Exception ex)
{
Log.Exception(this, ex, "Exception Thread processing queue item ", i);
result = RFProcessingResult.Error(new string[] { ex.Message }, ex is DbException || ex is TimeoutException);
}
// send result and all buffered events/instructions to external event manager (since
// we can't guarantee delivery order with MSMQ)
_eventSink.RaiseEvent(this, new RFProcessingFinishedEvent(i, result, sink.GetItems()), i.ProcessingKey);
//ts.Complete();
}
}
catch (Exception ex)
{
Log.Exception(this, ex, "Exception Thread processing queue item ", i);
}
}
public void Finished(RFWorkQueueItem i, RFProcessingResult result)
{
if (Filter(i))
{
Update(i.Item, GetState(result), null, result);
}
}
public RFProcessingFinishedEvent(RFWorkQueueItem item, RFProcessingResult result, RFWorkQueueItem[] queueItems)
{
Item = item;
Timestamp = DateTime.Now;
Result = result;
WorkQueueItems = queueItems;
}
public void CycleFinished(RFWorkQueueItem i, RFProcessingResult result)
{
if (i.ProcessingKey != null && _requests.ContainsKey(i.ProcessingKey) && i.Item is RFProcessInstruction)
{
_requests[i.ProcessingKey].CycleFinished((i.Item as RFProcessInstruction).ToString(), result);
}
}
// can start if none of its dependencies are running or about to run
private bool CanStart(RFWorkQueueItem i)
{
if (i.Item is RFGraphProcessInstruction)
{
var gpi = i.Item as RFGraphProcessInstruction;
if (_processDependencies.ContainsKey(gpi.ProcessName))
{
var depends = _processDependencies[gpi.ProcessName];
return(!_inProgress.Select(p => p.Item as RFProcessInstruction).Any(p => p != null && depends.Contains(p.ProcessName)) &&
!_readyQueue.Select(p => p.Item as RFProcessInstruction).Any(p => p != null && depends.Contains(p.ProcessName)));
}
}
else if (i.Item is RFProcessInstruction)
{
// queue up identical instructions
var pi = i.Item as RFProcessInstruction;
var existingForSameProcess = _inProgress.Select(p => p.Item as RFProcessInstruction).Where(p => p != null && p.ProcessName == pi.ProcessName)
.Union(_readyQueue.Select(p => p.Item as RFProcessInstruction).Where(p => p != null && p.ProcessName == pi.ProcessName));
if (existingForSameProcess.Any())
{
if (_exclusiveProcesses.Contains(pi.ProcessName))
{
// cannot run concurrently - queue
return(false);
}
return(!existingForSameProcess.Any(e => RFEngineProcessorParam.Equals(e.ExtractParam(), pi.ExtractParam()))); // catalog update on same key events will be queued
}
}
return(true);
}
public void Queued(RFWorkQueueItem i, long weight)
{
if (Filter(i))
{
Update(i.Item, DispatchState.Queued, weight, null);
}
}
public void CycleStarted(RFWorkQueueItem i, int cyclesRemaining)
{
if (i.ProcessingKey != null && _requests.ContainsKey(i.ProcessingKey) && i.Item is RFProcessInstruction)
{
_requests[i.ProcessingKey].CyclesRemaining(cyclesRemaining);
_requests[i.ProcessingKey].CycleStarted((i.Item as RFProcessInstruction).ToString());
}
}
private void ProcessingStarted(RFWorkQueueItem item)
{
if (item.Item is RFProcessInstruction)
{
_inProgress.Add(item);
RFStatic.Log.Debug(typeof(RFGraphDispatchQueue), "Processing started for {0} ({1} in progress)", item.Item, _inProgress.Count);
_dispatchStore.Started(item);
}
RefreshReadyQueue();
}
private void ProcessInstructionThread(RFWorkQueueItem i)
{
try
{
Log.Debug(this, "Started thread to process instruction {0}", i.Item as RFProcessInstruction);
var result = _context.Engine.Process(i.Item as RFInstruction, _context.GetProcessingContext(i.ProcessingKey, _instructionSink, _eventSink, this));
_eventSink.RaiseEvent(this, new RFProcessingFinishedEvent(i, result, null), i.ProcessingKey); // we do not store work queue items in-proc
}
catch (Exception ex)
{
Log.Exception(this, ex, "Exception Thread processing queue item ", i);
_eventSink.RaiseEvent(this, new RFProcessingFinishedEvent(i, RFProcessingResult.Error(new string[] { ex.Message }, false), null), i.ProcessingKey);
}
}
private static bool Filter(RFWorkQueueItem i)
{
if (i?.Item != null)
{
var type = i.Item.DispatchItemType();
switch (type)
{
case ItemType.NotSupported:
return(false);
default:
return(true);
}
}
return(false);
}
public void ProcessingFinished(RFWorkQueueItem i, RFProcessingResult result)
{
Monitor.Enter(_sync);
if (i.Item is RFProcessInstruction)
{
_inProgress.Remove(i);//.Item as RFProcessInstruction);
RFStatic.Log.Debug(typeof(RFGraphDispatchQueue), "Processing finished for {0} ({1} in progress)", i.Item, _inProgress.Count);
_dispatchStore.Finished(i, result);
}
RefreshReadyQueue();
RefreshInstructionCount();
if (!InProgress(i.ProcessingKey))
{
QueueItem(new RFWorkQueueItem {
Item = new RFRequestCompleted(), ProcessingKey = i.ProcessingKey
}); // notify monitor this request is done
}
Monitor.PulseAll(_sync); // inprogress finished and nothing left
Monitor.Exit(_sync);
}
protected override void ProcessQueueItem(RFWorkQueueItem item)
{
_workerQueue.Send(item);
}
protected override void ProcessQueueItem(RFWorkQueueItem item)
{
Task.Run(() => ProcessInstructionThread(item), _context.CancellationTokenSource.Token);
}
protected abstract void ProcessQueueItem(RFWorkQueueItem item);
public void QueueItem(RFWorkQueueItem i)
{
if (i.Item == null)
{
throw new RFSystemException(this, "Invalid empty work item received.");
}
Monitor.Enter(_sync);
// ignore if already running or queued (timers)
if (i.Item is RFProcessInstruction && (_inProgress.Any() || _readyQueue.Any()))
{
if (i.Item is RFGraphProcessInstruction)
{
// if one processor subscribes to multiple outputs of another processor duplicate
// instruction can happen
var gpi = i.Item as RFGraphProcessInstruction;
var running = _inProgress.Where(p => p.Item is RFGraphProcessInstruction).Select(p => p.Item as RFGraphProcessInstruction).ToList();
if (running.Any(r => r.Instance.Equals(gpi.Instance) && r.ProcessName == gpi.ProcessName))
{
RFStatic.Log.Debug(this, "Ignoring instruction {0} for as we have one running already", gpi);
return; // ignore already in progress
}
var ready = _readyQueue.Where(p => p.Item is RFGraphProcessInstruction).Select(p => p.Item as RFGraphProcessInstruction).ToList();
if (ready.Any(r => r.Instance.Equals(gpi.Instance) && r.ProcessName == gpi.ProcessName))
{
RFStatic.Log.Debug(this, "Ignoring instruction {0} for as we have one ready already", gpi);
return; // ignore already ready
}
}
// not sure this was used - probably not since this throws away an instruction rather than delays it
/*
* else
* {
* var pi = i.Item as RFProcessInstruction;
* if (pi.Event is RFIntervalEvent) // allow concurrent processing of multiple files or external triggers but not timers
* {
* if (_inProgress.Any())
* {
* var inProgress = _inProgress.Where(q => q.Item is RFProcessInstruction).Select(q => q.Item as RFProcessInstruction);
* if (inProgress.Any(q => q.ProcessName == pi.ProcessName))
* {
* return; // ignore already in progress - for example FTP taking longer than timed trigger
* }
* }
*
* if (_readyQueue.Any())
* {
* var queued = _readyQueue.Where(q => q.Item is RFProcessInstruction).Select(q => q.Item as RFProcessInstruction);
* if (queued.Any(q => q.ProcessName == pi.ProcessName))
* {
* return; // ignore already same process queued
* }
* }
* }
* }*/
}
if (i.Item is RFGraphProcessInstruction)
{
var gpi = i.Item as RFGraphProcessInstruction;
if (gpi.Instance != null && gpi.Instance.ValueDate.HasValue)
{
long datePart = (gpi.Instance.ValueDate.Value.ToYMD() - (long)20000000) * 1000000;
if (_processWeights.ContainsKey(gpi.ProcessName))
{
var weight = _processWeights[gpi.ProcessName];
var queueKey = datePart + weight;
if (_pendingInstructions.ContainsKey(queueKey))
{
var potentialDupes = _pendingInstructions[queueKey].Select(d => d.Item as RFGraphProcessInstruction);
// ignore duplicate
if (potentialDupes.Any(qi => qi.ProcessName.Equals(gpi.ProcessName) && qi.Instance.Equals(gpi.Instance)))
{
RFStatic.Log.Debug(this, "Ignoring instruction {0} for as we have one in the queue already", gpi);
Monitor.Exit(_sync);
return;
}
}
else
{
_pendingInstructions.Add(queueKey, new List <RFWorkQueueItem>());
}
_pendingInstructions[queueKey].Add(i);
_dispatchStore.Queued(i, queueKey);
}
else
{
RFStatic.Log.Info(this, "Assuming zero weighted for unweighted graph instruction {0}/{1}", gpi.ProcessName,
gpi.Instance.ToString());
if (!_pendingInstructions.ContainsKey(datePart))
{
_pendingInstructions.Add(datePart, new List <RFWorkQueueItem>());
}
_pendingInstructions[datePart].Add(i);
_dispatchStore.Queued(i, datePart);
}
}
else
{
RFStatic.Log.Warning(this, "Received graph instruction for process {0} without instance date.", gpi.ProcessName);
if (!_pendingInstructions.ContainsKey(0))
{
_pendingInstructions.Add(0, new List <RFWorkQueueItem>());
}
_pendingInstructions[0].Add(i); // no instance date
_dispatchStore.Queued(i, 0);
}
RefreshReadyQueue();
}
else if (i.Item is RFProcessInstruction)
{
var pi = i.Item as RFProcessInstruction;
if (!_pendingInstructions.ContainsKey(0))
{
_pendingInstructions.Add(0, new List <RFWorkQueueItem>());
}
// if we already have an identical pending instruction, we ignore (meaning we'll
// still queue if there's an identical one already *in progress*) the logic here is
// that if there's been multiple instances of the same event, we only process once
var potentialDupes = _pendingInstructions[0].Select(p => p.Item as RFProcessInstruction).Where(p => p != null && p.ProcessName == pi.ProcessName);
if (!potentialDupes.Any(d => RFEngineProcessorParam.Equals(d?.ExtractParam(), pi?.ExtractParam())))
{
_pendingInstructions[0].Add(i); // no instance date
_dispatchStore.Queued(i, 0);
RefreshReadyQueue();
}
/*else
* {
* RFStatic.Log.Debug(this, "Ignoring already pending RFProcessInstruction {0}", pi);
* }*/
}
else
{
_readyQueue.Add(i); // straight to readyqueue
_dispatchStore.Queued(i, 0);
Monitor.PulseAll(_sync);
}
RefreshInstructionCount();
Monitor.Exit(_sync);
}
protected void ProcessItem(RFWorkQueueItem item)
{
if (item != null)
{
try
{
var content = item.Item;
if (content is RFInstruction i)
{
if (i.ForceProcessLocally())
{
// processing results will go directly into our internal queues rather than be distributed
var result = _context.Engine.Process(content as RFInstruction, _context.GetProcessingContext(item.ProcessingKey, _instructionSink, _eventSink, this));
ProcessingFinished(item, result);
}
else
{
Log.Debug(this, "Received instruction {0}", content);
ProcessQueueItem(item);
}
}
else if (content is RFEvent)
{
if (!(content is RFIntervalEvent) && !((content is RFCatalogUpdateEvent) && (content as RFCatalogUpdateEvent).Key.Plane == RFPlane.Ephemeral))
{
Log.Debug(this, "Received event {0}", content);
}
if (content is RFProcessingFinishedEvent)
{
var fe = content as RFProcessingFinishedEvent;
RFStatic.Log.Debug(this, "Processing finished event for {0} with {1} events", fe.Item, (fe.WorkQueueItems ?? new RFWorkQueueItem[0]).Length);
// queue all resulting events and instructions
foreach (var wi in fe.WorkQueueItems ?? new RFWorkQueueItem[0])
{
if (wi.Item is RFEvent)
{
// react to events immediately as they might queue calculations
// and impact calculation order
_context.Engine.React(wi.Item as RFEvent, _context.GetProcessingContext(item.ProcessingKey, _instructionSink, _eventSink, this));
}
else
{
// queue process instructions
_dispatchQueue.QueueItem(wi);
}
}
// this will ensure all processing artifacts are queued before marking
// processing as complete (avoid finishing while events still pending)
ProcessingFinished(fe.Item, fe.Result);
}
_context.Engine.React(content as RFEvent, _context.GetProcessingContext(item.ProcessingKey, _instructionSink, _eventSink, this));
_dispatchQueue.ProcessingFinished(item, null);
}
}
catch (Exception ex)
{
Log.Exception(this, ex, "Exception processing queue item ", item);
}
}
}
private void ProcessingFinished(RFWorkQueueItem i, RFProcessingResult result)
{
_requestTracker.CycleFinished(i, result);
_dispatchQueue.ProcessingFinished(i, result);
}
