public void AssembleMachineForTrayProcessor(AtlasException atlasException, ErrorHandlingActivityMachine errorMachine) { if (atlasException.ExceptionOperationsGroup.Recovery is Reinitialize) { if (atlasException.Station is IReinitializable) { var reinitializableStation = atlasException.Station as IReinitializable; if (reinitializableStation.AllowsReinitialization) { LogService.Log(LogType.System, LogMessageType.Debug, GetType().Name, string.Format(CultureInfo.InvariantCulture, "Reinitialization on {0}. AllowReinitialization: {1}", atlasException.Station.Type, reinitializableStation.AllowsReinitialization)); //if this succeeds it will quit the machine AssembleMachineForReinitRecovery(errorMachine, errorMachine.Configuration.Data.Instrument as IInstrument, atlasException); } else { LogService.Log(LogType.System, LogMessageType.Error, GetType().Name, string.Format(CultureInfo.InvariantCulture, "Reinitialization not possible on {0}. AllowReinitialization: {1}", atlasException.Station.Type, reinitializableStation.AllowsReinitialization), atlasException); } } else { LogService.Log(LogType.System, LogMessageType.Error, GetType().Name, "Station is not IReinitializable. Reinitialization not possible on " + atlasException.Station.Type, atlasException); } } // the activities added by this will be executed only if reinitialize fails or is not performed. SetSeverityActivities(errorMachine, atlasException); }
public void SetDegradedModeActivity(DynamicActivityMachine machine, AtlasException e) { if (e.ExceptionOperationsGroup.DegradedOperation == null) { return; } IInstrument instrument = machine.Configuration.Data.Instrument; machine.AddActivity(new DynamicActivity("Instrument.Degrade", () => instrument.Degrade(e.ExceptionOperationsGroup.DegradedOperation))); }
protected Action HandleFailedAction(string name, AtlasException ex, bool handle = true) { return(() => { if (handle) { ex.Handle(); } LogService.Log(LogType.System, LogMessageType.Error, GetType().Name, string.Format(CultureInfo.InvariantCulture, "{0} Command Failed", name)); throw ex; }); }
protected override bool CompleteExecution(string explanation, Exception causeOfFault = null) { Exception ex = null; var errorCode = 0; if (HasDataContextKey("ErrorLevel")) { errorCode = DataContext.ErrorLevel; //Prevent parent machines from detecting the same error code DataContext.ErrorLevel = 0; } var station = DataContext.Station as IStation; if (causeOfFault != null) { if (!(causeOfFault is AtlasException)) { ex = new AtlasException(errorCode, station, causeOfFault); lock (_executionLock) { Fault = ex; } } else { ex = causeOfFault; } } else if (errorCode > 0) { ex = new AtlasException(errorCode, station); lock (_executionLock) { Fault = ex; } } if (ex != null) { lock (_executionLock) { _completionCause = CompletionCause.Faulted; } } return(base.CompleteExecution(explanation, ex)); }
/// <summary> /// Publishes an error notification. /// </summary> /// <param name="machine"></param> /// <param name="e"></param> protected void SetNotificationActivities(DynamicActivityMachine machine, AtlasException e) { if (e.Severity != ExceptionSeverityCategory.Warning && e.Severity != ExceptionSeverityCategory.Information) { if (e.Severity != ExceptionSeverityCategory.Unknown) { var instrumentManager = machine.Configuration.Data.InstrumentManager as IInstrumentManager; if (instrumentManager != null) { machine.AddActivity(new DynamicActivity("InstrumentManager.AddToErrorList", () => instrumentManager.SystemExceptions.Add(e))); } if (e.Severity == ExceptionSeverityCategory.Fatal) { machine.AddActivity(new DynamicActivity(MessageTokens.Instance[Request.CloseAllDialogs], () => MessageBus.Default.Notify("DialogEvent.RequestCloseAll", this, new NotificationEventArgs()))); } //TODO: this should use ShowErrorDialog once dialogs are refactored. Alerts dialog does not suppress others. machine.AddActivity(new DynamicActivity("MessageBus.ShowAlertDialog", () => MessageBus.Default.Notify(MessageTokens.Instance[Request.ShowAlertsDialog], e, new NotificationEventArgs <AtlasException>(e.FullDisplayLocalized, e)))); } _atlasLogger = new AtlasLogger(); _pubSub = PubSub.PubSub.GetInstance(_atlasLogger); var ventanaConnectMessage = new StatusUpdated { ErrorCode = e.Code, ErrorMessage = e.Description, }; machine.AddActivity(new DynamicActivity("PubSub.PublishStatus", () => _pubSub.Publish <IStatusUpdated>(ventanaConnectMessage))); } }
private static string GetErrorMachineName(AtlasException ae) { string name = string.Empty; switch (ae.Severity) { case ExceptionSeverityCategory.Standard: name = "StandardError-" + ae.Code; break; case ExceptionSeverityCategory.Severe: name = "SevereError-" + ae.Code; break; case ExceptionSeverityCategory.Critical: name = "CriticalError-" + ae.Code; break; case ExceptionSeverityCategory.Fatal: name = "FatalError-" + ae.Code; break; case ExceptionSeverityCategory.Warning: name = "Warning-" + ae.Code; break; case ExceptionSeverityCategory.Information: name = "Information-" + ae.Code; break; } if (ae.Station != null) { name = ae.Station.Name + ":" + name; } return(name); }
/// <summary> /// wires up the severity based builder definitions to the exception severity /// </summary> /// <param name="machine"></param> /// <param name="e"></param> protected abstract void SetSeverityActivities(DynamicActivityMachine machine, AtlasException e);
public void SetEmergencyStopInstrumentActivity(DynamicActivityMachine machine, AtlasException e) { IInstrumentManager instrumentManager = machine.Configuration.Data.InstrumentManager; machine.AddActivity(new DynamicActivity("InstrumentManager.StopSystem", () => instrumentManager.StopSystem(true))); }
public void AssembleMachineForCriticalError(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e) { var trayProcessor = e.Station as ITrayProcessor; SetDegradedModeActivity(activityMachine, e); // Redundant release for assurance activityMachine.UseFinalExitBehavior(new DynamicActivity("Release locks", () => trayProcessor.ReleaseLock(activityMachine))); // Only assemble a machine for an error if the station is running. activityMachine.AddFinishOrContinueCondition( new DynamicConstraint <IStation>("StationStoppedOrDisabled", e.Station, IsStationStoppedOrDisabled), new DynamicConstraint <IStation>("StationNotStoppedOrDisabled", e.Station, IsStationNotStoppedOrDisabled)); // Now get a resource lock for the error handling machine. activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Wait For Lock on {0}", trayProcessor.Name), () => trayProcessor.WaitForLock(activityMachine))); // Try to set the station's error and quit if it already had one set. activityMachine.AddActivity(new ActOnResultActivity <bool>("SetError", () => e.Station.SetError(e), false, () => activityMachine.Quit("SetError returned false"))); // Set the tray to error, which tells scheduler to leave it in the module. if (activityMachine.Tray != null) { activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Notify Tray ProcessError {0}", trayProcessor.Name), () => activityMachine.Tray.OnProcessingErrorOccurred(e))); activityMachine.AddActivity(new DynamicActivity(activityMachine.TrayHandler.Name + ".AbortFlowForTray", () => activityMachine.InstrumentManager.AbortFlowForTray(activityMachine.Tray))); } // Make the first activity to stop the station. activityMachine.SetActivityStop(); //If the station is disabled because the stop process failed, then we want to quit. activityMachine.AddQuitOrContinueCondition(new DynamicConstraint <ITrayProcessor>("StationState == StationState.Disabled", trayProcessor, s => s.State == StationState.Disabled), new DynamicConstraint <ITrayProcessor>("StationState == StationState.Stopped", trayProcessor, s => s.State == StationState.Stopped)); // Disable the station when it is finished processing and do NOT permit it to re-enable. activityMachine.AddActivity(new DynamicActivity(trayProcessor.Name + ".Disable", () => ((ISupportsDisabling)trayProcessor).Disable(AllowStationEnableAfterError))); activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Release Lock on {0}", trayProcessor.Name), () => trayProcessor.ReleaseLock(activityMachine))); }
/// <summary> /// If station has a tray then the currently running job for the tray will be aborted. /// This adds a trigger to the machine for the IsTrayDetected property and for the tray state. /// </summary> /// <param name="activityMachine"></param> /// <param name="instrument"></param> /// <param name="e"></param> /// <remarks>Requires externally established trigger for State on the station</remarks> protected void AssembleMachineForAbortTray(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e) { var trayProcessor = e.Station as ITrayProcessor; var trayHandler = e.Station as ITrayHandler; var trayDetector = e.Station as ITrayDetector; var tray = activityMachine.Tray; if (tray == null) { } if (trayHandler != null && trayDetector != null && tray != null) { activityMachine.UseRuntimeTrigger(new PropertyChangedTrigger("IsTrayDetected Changed", trayDetector, trayDetector.PropertyToString(() => trayDetector.IsTrayDetected))); activityMachine.UseRuntimeTrigger(new PropertyChangedTrigger("TrayStateChanged", tray, tray.PropertyToString(() => tray.State))); // Tell the machine to quit acting if the tray is lost. activityMachine.SetQuitOnTrayState(TrayState.Lost); activityMachine.AddActivity( new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Notify Tray Aborted {0}", activityMachine.TrayHandler.Name), () => activityMachine.Tray.OnAborted(TrayAbortType.Immediate))); activityMachine.AddActivity( new DynamicActivity(activityMachine.TrayHandler.Name + ".AbortFlowForTray", () => activityMachine.InstrumentManager.AbortFlowForTray(tray))); // Pause here until the station is done with the abort process and the tray has been retrieved. activityMachine.AddContinueCondition(new DynamicConstraint <IStation>(trayProcessor.Name, trayProcessor as IStation, s => s.State == StationState.Idle && !(s as ITrayDetector).IsTrayDetected)); } }
protected override void SetSeverityActivities(DynamicActivityMachine machine, AtlasException ex) { //Raise error dialog SetNotificationActivities(machine, ex); var stationMachine = machine as ErrorHandlingActivityMachine; if (stationMachine != null) { SetEmergencyStopInstrumentActivity(stationMachine, ex); } }
public void AssembleMachineForStandardError(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e) { var trayProcessor = e.Station as ITrayProcessor; AssembleMachineForQuitOnErrorEscalation(activityMachine, e); SetDegradedModeActivity(activityMachine, e); activityMachine.UseFinalExitBehavior(new DynamicActivity("Release locks", () => trayProcessor.ReleaseLock(activityMachine))); // Lock was obtained by machine CTOR, so it's safe to assemble the machine based on current station state. // Only assemble a machine for an error if the station is running. activityMachine.AddFinishOrContinueCondition( new DynamicConstraint <IStation>("StationStoppedOrDisabled", e.Station, IsStationStoppedOrDisabled), new DynamicConstraint <IStation>("StationNotStoppedOrDisabled", e.Station, IsStationNotStoppedOrDisabled)); // Try to set the station's error and quit if it already had one set. activityMachine.AddActivity(new ActOnResultActivity <bool>("SetError", () => e.Station.SetError(e), false, () => activityMachine.Quit("SetError returned false"))); // if tray present notify of a process anomaly occurred. if (activityMachine.Tray != null) { activityMachine.UseRuntimeTrigger(new PropertyChangedTrigger("TrayStateChanged", activityMachine.Tray, "State")); // Tell the machine to quit acting if the tray is lost. activityMachine.SetQuitOnTrayState(TrayState.Lost); activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Notify Tray ProcessAnomaly {0}" , trayProcessor.Name), () => activityMachine.Tray.OnProcessingAnomalyOccurred(e))); // Pause here until the station is done with the current process and the tray has been retrieved. activityMachine.AddContinueCondition(new DynamicConstraint <IStation>(trayProcessor.Name + " tray processing finished?", trayProcessor as IStation , s => s.State == StationState.Idle && !(s as ITrayDetector).IsTrayDetected)); } // Now get a resource lock for the error handling machine. activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Wait For Lock on {0}", trayProcessor.Name), () => trayProcessor.WaitForLock(activityMachine))); activityMachine.AddQuitOrContinueCondition(new DynamicConstraint <ITrayProcessor>("StationState == StationState.Disabled", trayProcessor, s => s.State == StationState.Disabled), new DynamicConstraint <ITrayProcessor>("StationState != StationState.Disabled", trayProcessor, s => s.State != StationState.Disabled)); // Disable the station once processing is done and allow user to re-enable. activityMachine.SetActivityDisable(AllowStationEnableAfterError); // Make this machine wait until the station reports that it is indeed disabled by disable(). activityMachine.SetPauseUntilStationStatus(StationState.Disabled); activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Release Lock on {0}", trayProcessor.Name), () => trayProcessor.ReleaseLock(activityMachine))); }
private void AssembleMachineForQuitOnErrorEscalation(DynamicActivityMachine machine, AtlasException e) { // This will trigger if either Error or Severity property notifies of a change. machine.UseRuntimeTrigger(new PropertyChangedTrigger("Station Error Severity Changed", e.Station, e.Station.PropertyToString(() => e.Station.Error) + "." + e.Station.PropertyToString(() => e.Station.Error.Severity))); // Quit at any time if the station's error escalates in severity. machine.UseAdditionalQuitCondition(new DynamicConstraint <IStation>("Station Error Severity Increased?", e.Station, (s) => s.Error != null && s.Error.Severity.IsGreaterThan(e.Severity))); }
protected override void SetSeverityActivities(DynamicActivityMachine machine, AtlasException e) { switch (e.Severity) { case ExceptionSeverityCategory.Fatal: AssembleMachineForFatalError(machine as ErrorHandlingActivityMachine, machine.Configuration.Data.Instrument as IInstrument, e); break; case ExceptionSeverityCategory.Critical: AssembleMachineForCriticalError(machine as ErrorHandlingActivityMachine, machine.Configuration.Data.Instrument as IInstrument, e); break; case ExceptionSeverityCategory.Severe: AssembleMachineForSevereError(machine as ErrorHandlingActivityMachine, machine.Configuration.Data.Instrument as IInstrument, e); break; case ExceptionSeverityCategory.Standard: AssembleMachineForStandardError(machine as ErrorHandlingActivityMachine, machine.Configuration.Data.Instrument as IInstrument, e); break; } }
public void AssembleMachineForFatalError(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e) { // No resource locking required for fatal //stop commands running on stations and shut down instrument. SetStopInstrumentActivity(activityMachine, e); }
public void AssembleMachineForReinitRecovery(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e) { var station = e.Station; var trayDetector = station as ITrayDetector; if (trayDetector != null) { activityMachine.UseRuntimeTrigger(new PropertyChangedTrigger("IsTrayDetected Changed", trayDetector, trayDetector.PropertyToString(() => trayDetector.IsTrayDetected))); } activityMachine.AddActivity(new DynamicActivity("Log Reinitialize", () => LogService.Log(GetType().Name, "Begin reinitialize activity."))); if (station is IReinitializable) { var reinitializableStation = station as IReinitializable; if (reinitializableStation.ReinitAttempted) { activityMachine.AddActivity(new DynamicActivity("Log Reinitialize", () => LogService.Log(GetType().Name, "Station already attempted Reinit"))); return; } activityMachine.AddActivity(new DynamicActivity("Reinit set initial states", () => (e.Station as IReinitializable).Reinit(e))); } else { activityMachine.AddActivity(new DynamicActivity("Log Reinitialize", () => LogService.Log(GetType().Name, "Station is not IReinitializable"))); return; } AssembleMachineForAbortTray(activityMachine, instrument, e); // Now get a resource lock for the remaining reinit operations. activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Wait For Lock on {0}", trayDetector.Name), () => trayDetector.WaitForLock(activityMachine))); activityMachine.AddConditionalActivity(new DynamicConstraint <IStation>("Station has no tray and is idle.", e.Station, (s => (s is IReinitializable && (s as IReinitializable).AllowsReinitialization) && //1. Module is IReinitializable ((s is ITrayHandler && !(s as ITrayHandler).IsTrayDetected) || (s is ITrayProcessor && !(s as ITrayProcessor).IsTrayDetected)) && //2. Tray is not present s.State == StationState.Idle)) //3. Module is idle , new DynamicActivity("Station.Stop", () => e.Station.Stop())); //this delays the start until the station state is stopped activityMachine.AddConditionalActivity(new DynamicConstraint <IStation>("Station is stopped.", e.Station, (s => s.State == StationState.Stopped)) , new DynamicActivity("Station.Reinitialize", () => e.Station.Start())); activityMachine.AddContinueCondition(new DynamicConstraint <IStation>("Is Processing? (after Start)", e.Station, (s => (!(e.Station is ITrayProcessor) || !(e.Station as ITrayProcessor).IsProcessing)))); activityMachine.AddActivity(new DynamicActivity("Station update Reinit State", () => { var reinitializableStation = e.Station as IReinitializable; if (reinitializableStation != null) { reinitializableStation.ReinitState = e.Station.State == StationState.Idle ? ReinitState.Success : ReinitState.Fail; } })); activityMachine.AddFinishOrContinueCondition( new DynamicConstraint <IStation>("Station reinitialize succeeded.", e.Station, (s => s.State == StationState.Idle)) , new DynamicConstraint <IStation>("Station reinitialize failed.", e.Station, (s => s.State != StationState.Idle)) ); activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Release Lock on {0}", trayDetector.Name), () => trayDetector.ReleaseLock(activityMachine))); }
public void AssembleMachineForSevereError(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e) { var trayProcessor = e.Station as ITrayProcessor; AssembleMachineForQuitOnErrorEscalation(activityMachine, e); SetDegradedModeActivity(activityMachine, e); activityMachine.UseFinalExitBehavior(new DynamicActivity("Release locks", () => trayProcessor.ReleaseLock(activityMachine))); // Only assemble a machine for an error if the station is running. activityMachine.AddFinishOrContinueCondition( new DynamicConstraint <IStation>("StationStoppedOrDisabled", e.Station, IsStationStoppedOrDisabled), new DynamicConstraint <IStation>("StationNotStoppedOrDisabled", e.Station, IsStationNotStoppedOrDisabled)); // Try to set the station's error and quit if it already had one set. activityMachine.AddActivity(new ActOnResultActivity <bool>("SetError", () => e.Station.SetError(e), false, () => activityMachine.Quit("SetError returned false"))); AssembleMachineForAbortTray(activityMachine, instrument, e); // Now get a resource lock for the error handling machine. activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Wait For Lock on {0}", trayProcessor.Name), () => trayProcessor.WaitForLock(activityMachine))); activityMachine.AddQuitOrContinueCondition(new DynamicConstraint <ITrayProcessor>("StationState == StationState.Disabled", trayProcessor, s => s.State == StationState.Disabled), new DynamicConstraint <ITrayProcessor>("StationState != StationState.Disabled", trayProcessor, s => s.State != StationState.Disabled)); // Disable the station when it is finished processing. activityMachine.SetActivityDisable(AllowStationEnableAfterError); activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Release Lock on {0}", trayProcessor.Name), () => trayProcessor.ReleaseLock(activityMachine))); }