// can start if none of its dependencies are running or about to run private bool CanStart(RFWorkQueueItem i) { if (i.Item is RFGraphProcessInstruction) { var gpi = i.Item as RFGraphProcessInstruction; if (_processDependencies.ContainsKey(gpi.ProcessName)) { var depends = _processDependencies[gpi.ProcessName]; return(!_inProgress.Select(p => p.Item as RFProcessInstruction).Any(p => p != null && depends.Contains(p.ProcessName)) && !_readyQueue.Select(p => p.Item as RFProcessInstruction).Any(p => p != null && depends.Contains(p.ProcessName))); } } else if (i.Item is RFProcessInstruction) { // queue up identical instructions var pi = i.Item as RFProcessInstruction; var existingForSameProcess = _inProgress.Select(p => p.Item as RFProcessInstruction).Where(p => p != null && p.ProcessName == pi.ProcessName) .Union(_readyQueue.Select(p => p.Item as RFProcessInstruction).Where(p => p != null && p.ProcessName == pi.ProcessName)); if (existingForSameProcess.Any()) { if (_exclusiveProcesses.Contains(pi.ProcessName)) { // cannot run concurrently - queue return(false); } return(!existingForSameProcess.Any(e => RFEngineProcessorParam.Equals(e.ExtractParam(), pi.ExtractParam()))); // catalog update on same key events will be queued } } return(true); }
/// <summary> /// Do not override or call base when you do /// </summary> public virtual void Initialize(RFEngineProcessorParam p, IRFProcessingContext context, RFKeyDomain keyDomain, string processName) { InstanceParams = p?.ConvertTo <P>(); Context = context; KeyDomain = keyDomain; ProcessName = processName; _isCancelling = false; _processEntry = null; Log = new RFProcessLog(Context.SystemLog, context.UserLog, this); }
public static bool Equals(RFEngineProcessorParam l, RFEngineProcessorParam r) { if (l == null && r == null) { return(true); } else if (l != null && r != null) { return(l.Equals(r)); } return(false); }
public RFParamProcessInstruction(string processName, RFEngineProcessorParam param) : base(processName) { Param = param; }
public void QueueItem(RFWorkQueueItem i) { if (i.Item == null) { throw new RFSystemException(this, "Invalid empty work item received."); } Monitor.Enter(_sync); // ignore if already running or queued (timers) if (i.Item is RFProcessInstruction && (_inProgress.Any() || _readyQueue.Any())) { if (i.Item is RFGraphProcessInstruction) { // if one processor subscribes to multiple outputs of another processor duplicate // instruction can happen var gpi = i.Item as RFGraphProcessInstruction; var running = _inProgress.Where(p => p.Item is RFGraphProcessInstruction).Select(p => p.Item as RFGraphProcessInstruction).ToList(); if (running.Any(r => r.Instance.Equals(gpi.Instance) && r.ProcessName == gpi.ProcessName)) { RFStatic.Log.Debug(this, "Ignoring instruction {0} for as we have one running already", gpi); return; // ignore already in progress } var ready = _readyQueue.Where(p => p.Item is RFGraphProcessInstruction).Select(p => p.Item as RFGraphProcessInstruction).ToList(); if (ready.Any(r => r.Instance.Equals(gpi.Instance) && r.ProcessName == gpi.ProcessName)) { RFStatic.Log.Debug(this, "Ignoring instruction {0} for as we have one ready already", gpi); return; // ignore already ready } } // not sure this was used - probably not since this throws away an instruction rather than delays it /* * else * { * var pi = i.Item as RFProcessInstruction; * if (pi.Event is RFIntervalEvent) // allow concurrent processing of multiple files or external triggers but not timers * { * if (_inProgress.Any()) * { * var inProgress = _inProgress.Where(q => q.Item is RFProcessInstruction).Select(q => q.Item as RFProcessInstruction); * if (inProgress.Any(q => q.ProcessName == pi.ProcessName)) * { * return; // ignore already in progress - for example FTP taking longer than timed trigger * } * } * * if (_readyQueue.Any()) * { * var queued = _readyQueue.Where(q => q.Item is RFProcessInstruction).Select(q => q.Item as RFProcessInstruction); * if (queued.Any(q => q.ProcessName == pi.ProcessName)) * { * return; // ignore already same process queued * } * } * } * }*/ } if (i.Item is RFGraphProcessInstruction) { var gpi = i.Item as RFGraphProcessInstruction; if (gpi.Instance != null && gpi.Instance.ValueDate.HasValue) { long datePart = (gpi.Instance.ValueDate.Value.ToYMD() - (long)20000000) * 1000000; if (_processWeights.ContainsKey(gpi.ProcessName)) { var weight = _processWeights[gpi.ProcessName]; var queueKey = datePart + weight; if (_pendingInstructions.ContainsKey(queueKey)) { var potentialDupes = _pendingInstructions[queueKey].Select(d => d.Item as RFGraphProcessInstruction); // ignore duplicate if (potentialDupes.Any(qi => qi.ProcessName.Equals(gpi.ProcessName) && qi.Instance.Equals(gpi.Instance))) { RFStatic.Log.Debug(this, "Ignoring instruction {0} for as we have one in the queue already", gpi); Monitor.Exit(_sync); return; } } else { _pendingInstructions.Add(queueKey, new List <RFWorkQueueItem>()); } _pendingInstructions[queueKey].Add(i); _dispatchStore.Queued(i, queueKey); } else { RFStatic.Log.Info(this, "Assuming zero weighted for unweighted graph instruction {0}/{1}", gpi.ProcessName, gpi.Instance.ToString()); if (!_pendingInstructions.ContainsKey(datePart)) { _pendingInstructions.Add(datePart, new List <RFWorkQueueItem>()); } _pendingInstructions[datePart].Add(i); _dispatchStore.Queued(i, datePart); } } else { RFStatic.Log.Warning(this, "Received graph instruction for process {0} without instance date.", gpi.ProcessName); if (!_pendingInstructions.ContainsKey(0)) { _pendingInstructions.Add(0, new List <RFWorkQueueItem>()); } _pendingInstructions[0].Add(i); // no instance date _dispatchStore.Queued(i, 0); } RefreshReadyQueue(); } else if (i.Item is RFProcessInstruction) { var pi = i.Item as RFProcessInstruction; if (!_pendingInstructions.ContainsKey(0)) { _pendingInstructions.Add(0, new List <RFWorkQueueItem>()); } // if we already have an identical pending instruction, we ignore (meaning we'll // still queue if there's an identical one already *in progress*) the logic here is // that if there's been multiple instances of the same event, we only process once var potentialDupes = _pendingInstructions[0].Select(p => p.Item as RFProcessInstruction).Where(p => p != null && p.ProcessName == pi.ProcessName); if (!potentialDupes.Any(d => RFEngineProcessorParam.Equals(d?.ExtractParam(), pi?.ExtractParam()))) { _pendingInstructions[0].Add(i); // no instance date _dispatchStore.Queued(i, 0); RefreshReadyQueue(); } /*else * { * RFStatic.Log.Debug(this, "Ignoring already pending RFProcessInstruction {0}", pi); * }*/ } else { _readyQueue.Add(i); // straight to readyqueue _dispatchStore.Queued(i, 0); Monitor.PulseAll(_sync); } RefreshInstructionCount(); Monitor.Exit(_sync); }
public RFProcessingResult RunInstance(IRFEngineProcessor processorInstance, RFInstruction instruction, IRFProcessingContext context) { var pi = instruction as RFProcessInstruction; if (pi != null) { var sw = Stopwatch.StartNew(); RFEngineProcessorParam instanceParams = null; try { instanceParams = Config.InstanceParams(instruction); processorInstance.Initialize(instanceParams, context, KeyDomain, Config.Name); var result = processorInstance.Process(); result.AddMessages(processorInstance.Log.GetErrors()); if ((result.WorkDone || result.IsError) && !(processorInstance is RFSchedulerProcessor)) { context.SystemLog.LogProcess(this, processorInstance.GetProcessEntry() ?? new RFProcessEntry { GraphInstance = (instanceParams as RFEngineProcessorGraphInstanceParam)?.Instance, GraphName = (processorInstance as RFGraphProcess)?.GraphName, IOTime = 0, ProcessingTime = sw.ElapsedMilliseconds, Message = String.Join("\r\n", result.Messages), ProcessName = Config.Name, Success = !result.IsError, NumUpdates = 1 }); context.SystemLog.Info(this, "Run process {0}{1} in {2}ms", Config.Name, instanceParams != null ? String.Format(" ({0})", instanceParams) : String.Empty, sw.ElapsedMilliseconds); } return(result); } catch (RFLogicException ex) // soft exception (incorrect file etc.) { context.UserLog.LogEntry(new RFUserLogEntry { Action = "Warning", IsWarning = true, Description = String.Format("Calculation error: {0}", ex.Message), IsUserAction = false, Processor = Config.Name, Username = "******", Area = null, ValueDate = RFDate.NullDate }); context.SystemLog.Warning(this, "Logic Exception on process {0}: {1}", Config.Name, ex.Message); context.SystemLog.LogProcess(this, processorInstance.GetProcessEntry() ?? new RFProcessEntry { GraphInstance = (instanceParams as RFEngineProcessorGraphInstanceParam)?.Instance, GraphName = (processorInstance as RFGraphProcess)?.GraphName, IOTime = 0, ProcessingTime = sw.ElapsedMilliseconds, Message = ex.Message, ProcessName = Config.Name, Success = false, NumUpdates = 0 }); var result = new RFProcessingResult { IsError = true, Messages = new SortedSet <string>(processorInstance?.Log?.GetErrors() ?? new string[0]), WorkDone = false, ShouldRetry = false, UpdatedKeys = new System.Collections.Generic.List <RFCatalogKey>() }; result.AddMessage(ex.Message); return(result); } catch (Exception ex) // hard exception - system, null etc. { context.SystemLog.LogProcess(this, processorInstance?.GetProcessEntry() ?? new RFProcessEntry { GraphInstance = (instanceParams as RFEngineProcessorGraphInstanceParam)?.Instance, GraphName = (processorInstance as RFGraphProcess)?.GraphName, IOTime = 0, ProcessingTime = sw.ElapsedMilliseconds, Message = ex.Message, ProcessName = Config.Name, Success = false, NumUpdates = 0 }); throw; } } throw new RFSystemException(this, "Cannot process empty instruction"); }