// can start if none of its dependencies are running or about to run
private bool CanStart(RFWorkQueueItem i)
{
if (i.Item is RFGraphProcessInstruction)
{
var gpi = i.Item as RFGraphProcessInstruction;
if (_processDependencies.ContainsKey(gpi.ProcessName))
{
var depends = _processDependencies[gpi.ProcessName];
return(!_inProgress.Select(p => p.Item as RFProcessInstruction).Any(p => p != null && depends.Contains(p.ProcessName)) &&
!_readyQueue.Select(p => p.Item as RFProcessInstruction).Any(p => p != null && depends.Contains(p.ProcessName)));
}
}
else if (i.Item is RFProcessInstruction)
{
// queue up identical instructions
var pi = i.Item as RFProcessInstruction;
var existingForSameProcess = _inProgress.Select(p => p.Item as RFProcessInstruction).Where(p => p != null && p.ProcessName == pi.ProcessName)
.Union(_readyQueue.Select(p => p.Item as RFProcessInstruction).Where(p => p != null && p.ProcessName == pi.ProcessName));
if (existingForSameProcess.Any())
{
if (_exclusiveProcesses.Contains(pi.ProcessName))
{
// cannot run concurrently - queue
return(false);
}
return(!existingForSameProcess.Any(e => RFEngineProcessorParam.Equals(e.ExtractParam(), pi.ExtractParam()))); // catalog update on same key events will be queued
}
}
return(true);
}
/// <summary>
/// Do not override or call base when you do
/// </summary>
public virtual void Initialize(RFEngineProcessorParam p, IRFProcessingContext context, RFKeyDomain keyDomain, string processName)
{
InstanceParams = p?.ConvertTo <P>();
Context = context;
KeyDomain = keyDomain;
ProcessName = processName;
_isCancelling = false;
_processEntry = null;
Log = new RFProcessLog(Context.SystemLog, context.UserLog, this);
}
public static bool Equals(RFEngineProcessorParam l, RFEngineProcessorParam r)
{
if (l == null && r == null)
{
return(true);
}
else if (l != null && r != null)
{
return(l.Equals(r));
}
return(false);
}
public RFParamProcessInstruction(string processName, RFEngineProcessorParam param) : base(processName)
{
Param = param;
}
public void QueueItem(RFWorkQueueItem i)
{
if (i.Item == null)
{
throw new RFSystemException(this, "Invalid empty work item received.");
}
Monitor.Enter(_sync);
// ignore if already running or queued (timers)
if (i.Item is RFProcessInstruction && (_inProgress.Any() || _readyQueue.Any()))
{
if (i.Item is RFGraphProcessInstruction)
{
// if one processor subscribes to multiple outputs of another processor duplicate
// instruction can happen
var gpi = i.Item as RFGraphProcessInstruction;
var running = _inProgress.Where(p => p.Item is RFGraphProcessInstruction).Select(p => p.Item as RFGraphProcessInstruction).ToList();
if (running.Any(r => r.Instance.Equals(gpi.Instance) && r.ProcessName == gpi.ProcessName))
{
RFStatic.Log.Debug(this, "Ignoring instruction {0} for as we have one running already", gpi);
return; // ignore already in progress
}
var ready = _readyQueue.Where(p => p.Item is RFGraphProcessInstruction).Select(p => p.Item as RFGraphProcessInstruction).ToList();
if (ready.Any(r => r.Instance.Equals(gpi.Instance) && r.ProcessName == gpi.ProcessName))
{
RFStatic.Log.Debug(this, "Ignoring instruction {0} for as we have one ready already", gpi);
return; // ignore already ready
}
}
// not sure this was used - probably not since this throws away an instruction rather than delays it
/*
* else
* {
* var pi = i.Item as RFProcessInstruction;
* if (pi.Event is RFIntervalEvent) // allow concurrent processing of multiple files or external triggers but not timers
* {
* if (_inProgress.Any())
* {
* var inProgress = _inProgress.Where(q => q.Item is RFProcessInstruction).Select(q => q.Item as RFProcessInstruction);
* if (inProgress.Any(q => q.ProcessName == pi.ProcessName))
* {
* return; // ignore already in progress - for example FTP taking longer than timed trigger
* }
* }
*
* if (_readyQueue.Any())
* {
* var queued = _readyQueue.Where(q => q.Item is RFProcessInstruction).Select(q => q.Item as RFProcessInstruction);
* if (queued.Any(q => q.ProcessName == pi.ProcessName))
* {
* return; // ignore already same process queued
* }
* }
* }
* }*/
}
if (i.Item is RFGraphProcessInstruction)
{
var gpi = i.Item as RFGraphProcessInstruction;
if (gpi.Instance != null && gpi.Instance.ValueDate.HasValue)
{
long datePart = (gpi.Instance.ValueDate.Value.ToYMD() - (long)20000000) * 1000000;
if (_processWeights.ContainsKey(gpi.ProcessName))
{
var weight = _processWeights[gpi.ProcessName];
var queueKey = datePart + weight;
if (_pendingInstructions.ContainsKey(queueKey))
{
var potentialDupes = _pendingInstructions[queueKey].Select(d => d.Item as RFGraphProcessInstruction);
// ignore duplicate
if (potentialDupes.Any(qi => qi.ProcessName.Equals(gpi.ProcessName) && qi.Instance.Equals(gpi.Instance)))
{
RFStatic.Log.Debug(this, "Ignoring instruction {0} for as we have one in the queue already", gpi);
Monitor.Exit(_sync);
return;
}
}
else
{
_pendingInstructions.Add(queueKey, new List <RFWorkQueueItem>());
}
_pendingInstructions[queueKey].Add(i);
_dispatchStore.Queued(i, queueKey);
}
else
{
RFStatic.Log.Info(this, "Assuming zero weighted for unweighted graph instruction {0}/{1}", gpi.ProcessName,
gpi.Instance.ToString());
if (!_pendingInstructions.ContainsKey(datePart))
{
_pendingInstructions.Add(datePart, new List <RFWorkQueueItem>());
}
_pendingInstructions[datePart].Add(i);
_dispatchStore.Queued(i, datePart);
}
}
else
{
RFStatic.Log.Warning(this, "Received graph instruction for process {0} without instance date.", gpi.ProcessName);
if (!_pendingInstructions.ContainsKey(0))
{
_pendingInstructions.Add(0, new List <RFWorkQueueItem>());
}
_pendingInstructions[0].Add(i); // no instance date
_dispatchStore.Queued(i, 0);
}
RefreshReadyQueue();
}
else if (i.Item is RFProcessInstruction)
{
var pi = i.Item as RFProcessInstruction;
if (!_pendingInstructions.ContainsKey(0))
{
_pendingInstructions.Add(0, new List <RFWorkQueueItem>());
}
// if we already have an identical pending instruction, we ignore (meaning we'll
// still queue if there's an identical one already *in progress*) the logic here is
// that if there's been multiple instances of the same event, we only process once
var potentialDupes = _pendingInstructions[0].Select(p => p.Item as RFProcessInstruction).Where(p => p != null && p.ProcessName == pi.ProcessName);
if (!potentialDupes.Any(d => RFEngineProcessorParam.Equals(d?.ExtractParam(), pi?.ExtractParam())))
{
_pendingInstructions[0].Add(i); // no instance date
_dispatchStore.Queued(i, 0);
RefreshReadyQueue();
}
/*else
* {
* RFStatic.Log.Debug(this, "Ignoring already pending RFProcessInstruction {0}", pi);
* }*/
}
else
{
_readyQueue.Add(i); // straight to readyqueue
_dispatchStore.Queued(i, 0);
Monitor.PulseAll(_sync);
}
RefreshInstructionCount();
Monitor.Exit(_sync);
}
public RFProcessingResult RunInstance(IRFEngineProcessor processorInstance, RFInstruction instruction, IRFProcessingContext context)
{
var pi = instruction as RFProcessInstruction;
if (pi != null)
{
var sw = Stopwatch.StartNew();
RFEngineProcessorParam instanceParams = null;
try
{
instanceParams = Config.InstanceParams(instruction);
processorInstance.Initialize(instanceParams, context, KeyDomain, Config.Name);
var result = processorInstance.Process();
result.AddMessages(processorInstance.Log.GetErrors());
if ((result.WorkDone || result.IsError) && !(processorInstance is RFSchedulerProcessor))
{
context.SystemLog.LogProcess(this, processorInstance.GetProcessEntry() ?? new RFProcessEntry
{
GraphInstance = (instanceParams as RFEngineProcessorGraphInstanceParam)?.Instance,
GraphName = (processorInstance as RFGraphProcess)?.GraphName,
IOTime = 0,
ProcessingTime = sw.ElapsedMilliseconds,
Message = String.Join("\r\n", result.Messages),
ProcessName = Config.Name,
Success = !result.IsError,
NumUpdates = 1
});
context.SystemLog.Info(this, "Run process {0}{1} in {2}ms", Config.Name, instanceParams != null ? String.Format(" ({0})", instanceParams) : String.Empty, sw.ElapsedMilliseconds);
}
return(result);
}
catch (RFLogicException ex) // soft exception (incorrect file etc.)
{
context.UserLog.LogEntry(new RFUserLogEntry
{
Action = "Warning",
IsWarning = true,
Description = String.Format("Calculation error: {0}", ex.Message),
IsUserAction = false,
Processor = Config.Name,
Username = "******",
Area = null,
ValueDate = RFDate.NullDate
});
context.SystemLog.Warning(this, "Logic Exception on process {0}: {1}", Config.Name, ex.Message);
context.SystemLog.LogProcess(this, processorInstance.GetProcessEntry() ?? new RFProcessEntry
{
GraphInstance = (instanceParams as RFEngineProcessorGraphInstanceParam)?.Instance,
GraphName = (processorInstance as RFGraphProcess)?.GraphName,
IOTime = 0,
ProcessingTime = sw.ElapsedMilliseconds,
Message = ex.Message,
ProcessName = Config.Name,
Success = false,
NumUpdates = 0
});
var result = new RFProcessingResult
{
IsError = true,
Messages = new SortedSet <string>(processorInstance?.Log?.GetErrors() ?? new string[0]),
WorkDone = false,
ShouldRetry = false,
UpdatedKeys = new System.Collections.Generic.List <RFCatalogKey>()
};
result.AddMessage(ex.Message);
return(result);
}
catch (Exception ex) // hard exception - system, null etc.
{
context.SystemLog.LogProcess(this, processorInstance?.GetProcessEntry() ?? new RFProcessEntry
{
GraphInstance = (instanceParams as RFEngineProcessorGraphInstanceParam)?.Instance,
GraphName = (processorInstance as RFGraphProcess)?.GraphName,
IOTime = 0,
ProcessingTime = sw.ElapsedMilliseconds,
Message = ex.Message,
ProcessName = Config.Name,
Success = false,
NumUpdates = 0
});
throw;
}
}
throw new RFSystemException(this, "Cannot process empty instruction");
}
