public void AssembleMachineForTrayProcessor(AtlasException atlasException, ErrorHandlingActivityMachine errorMachine)
{
if (atlasException.ExceptionOperationsGroup.Recovery is Reinitialize)
{
if (atlasException.Station is IReinitializable)
{
var reinitializableStation = atlasException.Station as IReinitializable;
if (reinitializableStation.AllowsReinitialization)
{
LogService.Log(LogType.System, LogMessageType.Debug, GetType().Name,
string.Format(CultureInfo.InvariantCulture, "Reinitialization on {0}. AllowReinitialization: {1}",
atlasException.Station.Type, reinitializableStation.AllowsReinitialization));
//if this succeeds it will quit the machine
AssembleMachineForReinitRecovery(errorMachine, errorMachine.Configuration.Data.Instrument as IInstrument, atlasException);
}
else
{
LogService.Log(LogType.System, LogMessageType.Error, GetType().Name,
string.Format(CultureInfo.InvariantCulture, "Reinitialization not possible on {0}. AllowReinitialization: {1}",
atlasException.Station.Type, reinitializableStation.AllowsReinitialization), atlasException);
}
}
else
{
LogService.Log(LogType.System, LogMessageType.Error, GetType().Name, "Station is not IReinitializable. Reinitialization not possible on " + atlasException.Station.Type, atlasException);
}
}
// the activities added by this will be executed only if reinitialize fails or is not performed.
SetSeverityActivities(errorMachine, atlasException);
}
public void SetDegradedModeActivity(DynamicActivityMachine machine, AtlasException e)
{
if (e.ExceptionOperationsGroup.DegradedOperation == null)
{
return;
}
IInstrument instrument = machine.Configuration.Data.Instrument;
machine.AddActivity(new DynamicActivity("Instrument.Degrade", () => instrument.Degrade(e.ExceptionOperationsGroup.DegradedOperation)));
}
protected Action HandleFailedAction(string name, AtlasException ex, bool handle = true)
{
return(() =>
{
if (handle)
{
ex.Handle();
}
LogService.Log(LogType.System, LogMessageType.Error, GetType().Name,
string.Format(CultureInfo.InvariantCulture, "{0} Command Failed", name));
throw ex;
});
}
protected override bool CompleteExecution(string explanation, Exception causeOfFault = null)
{
Exception ex = null;
var errorCode = 0;
if (HasDataContextKey("ErrorLevel"))
{
errorCode = DataContext.ErrorLevel;
//Prevent parent machines from detecting the same error code
DataContext.ErrorLevel = 0;
}
var station = DataContext.Station as IStation;
if (causeOfFault != null)
{
if (!(causeOfFault is AtlasException))
{
ex = new AtlasException(errorCode, station, causeOfFault);
lock (_executionLock)
{
Fault = ex;
}
}
else
{
ex = causeOfFault;
}
}
else if (errorCode > 0)
{
ex = new AtlasException(errorCode, station);
lock (_executionLock)
{
Fault = ex;
}
}
if (ex != null)
{
lock (_executionLock)
{
_completionCause = CompletionCause.Faulted;
}
}
return(base.CompleteExecution(explanation, ex));
}
/// <summary>
/// Publishes an error notification.
/// </summary>
/// <param name="machine"></param>
/// <param name="e"></param>
protected void SetNotificationActivities(DynamicActivityMachine machine, AtlasException e)
{
if (e.Severity != ExceptionSeverityCategory.Warning &&
e.Severity != ExceptionSeverityCategory.Information)
{
if (e.Severity != ExceptionSeverityCategory.Unknown)
{
var instrumentManager = machine.Configuration.Data.InstrumentManager as IInstrumentManager;
if (instrumentManager != null)
{
machine.AddActivity(new DynamicActivity("InstrumentManager.AddToErrorList",
() => instrumentManager.SystemExceptions.Add(e)));
}
if (e.Severity == ExceptionSeverityCategory.Fatal)
{
machine.AddActivity(new DynamicActivity(MessageTokens.Instance[Request.CloseAllDialogs],
() => MessageBus.Default.Notify("DialogEvent.RequestCloseAll", this,
new NotificationEventArgs())));
}
//TODO: this should use ShowErrorDialog once dialogs are refactored. Alerts dialog does not suppress others.
machine.AddActivity(new DynamicActivity("MessageBus.ShowAlertDialog",
() => MessageBus.Default.Notify(MessageTokens.Instance[Request.ShowAlertsDialog], e,
new NotificationEventArgs <AtlasException>(e.FullDisplayLocalized, e))));
}
_atlasLogger = new AtlasLogger();
_pubSub = PubSub.PubSub.GetInstance(_atlasLogger);
var ventanaConnectMessage = new StatusUpdated
{
ErrorCode = e.Code,
ErrorMessage = e.Description,
};
machine.AddActivity(new DynamicActivity("PubSub.PublishStatus", () => _pubSub.Publish <IStatusUpdated>(ventanaConnectMessage)));
}
}
private static string GetErrorMachineName(AtlasException ae)
{
string name = string.Empty;
switch (ae.Severity)
{
case ExceptionSeverityCategory.Standard:
name = "StandardError-" + ae.Code;
break;
case ExceptionSeverityCategory.Severe:
name = "SevereError-" + ae.Code;
break;
case ExceptionSeverityCategory.Critical:
name = "CriticalError-" + ae.Code;
break;
case ExceptionSeverityCategory.Fatal:
name = "FatalError-" + ae.Code;
break;
case ExceptionSeverityCategory.Warning:
name = "Warning-" + ae.Code;
break;
case ExceptionSeverityCategory.Information:
name = "Information-" + ae.Code;
break;
}
if (ae.Station != null)
{
name = ae.Station.Name + ":" + name;
}
return(name);
}
/// <summary> /// wires up the severity based builder definitions to the exception severity /// </summary> /// <param name="machine"></param> /// <param name="e"></param> protected abstract void SetSeverityActivities(DynamicActivityMachine machine, AtlasException e);
public void SetEmergencyStopInstrumentActivity(DynamicActivityMachine machine, AtlasException e)
{
IInstrumentManager instrumentManager = machine.Configuration.Data.InstrumentManager;
machine.AddActivity(new DynamicActivity("InstrumentManager.StopSystem", () => instrumentManager.StopSystem(true)));
}
public void AssembleMachineForCriticalError(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e)
{
var trayProcessor = e.Station as ITrayProcessor;
SetDegradedModeActivity(activityMachine, e);
// Redundant release for assurance
activityMachine.UseFinalExitBehavior(new DynamicActivity("Release locks", () => trayProcessor.ReleaseLock(activityMachine)));
// Only assemble a machine for an error if the station is running.
activityMachine.AddFinishOrContinueCondition(
new DynamicConstraint <IStation>("StationStoppedOrDisabled", e.Station, IsStationStoppedOrDisabled),
new DynamicConstraint <IStation>("StationNotStoppedOrDisabled", e.Station, IsStationNotStoppedOrDisabled));
// Now get a resource lock for the error handling machine.
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Wait For Lock on {0}", trayProcessor.Name),
() => trayProcessor.WaitForLock(activityMachine)));
// Try to set the station's error and quit if it already had one set.
activityMachine.AddActivity(new ActOnResultActivity <bool>("SetError", () => e.Station.SetError(e), false, () => activityMachine.Quit("SetError returned false")));
// Set the tray to error, which tells scheduler to leave it in the module.
if (activityMachine.Tray != null)
{
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Notify Tray ProcessError {0}", trayProcessor.Name),
() => activityMachine.Tray.OnProcessingErrorOccurred(e)));
activityMachine.AddActivity(new DynamicActivity(activityMachine.TrayHandler.Name + ".AbortFlowForTray",
() => activityMachine.InstrumentManager.AbortFlowForTray(activityMachine.Tray)));
}
// Make the first activity to stop the station.
activityMachine.SetActivityStop();
//If the station is disabled because the stop process failed, then we want to quit.
activityMachine.AddQuitOrContinueCondition(new DynamicConstraint <ITrayProcessor>("StationState == StationState.Disabled",
trayProcessor, s => s.State == StationState.Disabled), new DynamicConstraint <ITrayProcessor>("StationState == StationState.Stopped",
trayProcessor, s => s.State == StationState.Stopped));
// Disable the station when it is finished processing and do NOT permit it to re-enable.
activityMachine.AddActivity(new DynamicActivity(trayProcessor.Name + ".Disable", () => ((ISupportsDisabling)trayProcessor).Disable(AllowStationEnableAfterError)));
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Release Lock on {0}", trayProcessor.Name),
() => trayProcessor.ReleaseLock(activityMachine)));
}
/// <summary>
/// If station has a tray then the currently running job for the tray will be aborted.
/// This adds a trigger to the machine for the IsTrayDetected property and for the tray state.
/// </summary>
/// <param name="activityMachine"></param>
/// <param name="instrument"></param>
/// <param name="e"></param>
/// <remarks>Requires externally established trigger for State on the station</remarks>
protected void AssembleMachineForAbortTray(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e)
{
var trayProcessor = e.Station as ITrayProcessor;
var trayHandler = e.Station as ITrayHandler;
var trayDetector = e.Station as ITrayDetector;
var tray = activityMachine.Tray;
if (tray == null)
{
}
if (trayHandler != null && trayDetector != null && tray != null)
{
activityMachine.UseRuntimeTrigger(new PropertyChangedTrigger("IsTrayDetected Changed", trayDetector,
trayDetector.PropertyToString(() => trayDetector.IsTrayDetected)));
activityMachine.UseRuntimeTrigger(new PropertyChangedTrigger("TrayStateChanged", tray, tray.PropertyToString(() => tray.State)));
// Tell the machine to quit acting if the tray is lost.
activityMachine.SetQuitOnTrayState(TrayState.Lost);
activityMachine.AddActivity(
new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Notify Tray Aborted {0}", activityMachine.TrayHandler.Name),
() => activityMachine.Tray.OnAborted(TrayAbortType.Immediate)));
activityMachine.AddActivity(
new DynamicActivity(activityMachine.TrayHandler.Name + ".AbortFlowForTray",
() => activityMachine.InstrumentManager.AbortFlowForTray(tray)));
// Pause here until the station is done with the abort process and the tray has been retrieved.
activityMachine.AddContinueCondition(new DynamicConstraint <IStation>(trayProcessor.Name, trayProcessor as IStation,
s => s.State == StationState.Idle && !(s as ITrayDetector).IsTrayDetected));
}
}
protected override void SetSeverityActivities(DynamicActivityMachine machine, AtlasException ex)
{
//Raise error dialog
SetNotificationActivities(machine, ex);
var stationMachine = machine as ErrorHandlingActivityMachine;
if (stationMachine != null)
{
SetEmergencyStopInstrumentActivity(stationMachine, ex);
}
}
public void AssembleMachineForStandardError(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e)
{
var trayProcessor = e.Station as ITrayProcessor;
AssembleMachineForQuitOnErrorEscalation(activityMachine, e);
SetDegradedModeActivity(activityMachine, e);
activityMachine.UseFinalExitBehavior(new DynamicActivity("Release locks", () => trayProcessor.ReleaseLock(activityMachine)));
// Lock was obtained by machine CTOR, so it's safe to assemble the machine based on current station state.
// Only assemble a machine for an error if the station is running.
activityMachine.AddFinishOrContinueCondition(
new DynamicConstraint <IStation>("StationStoppedOrDisabled", e.Station, IsStationStoppedOrDisabled),
new DynamicConstraint <IStation>("StationNotStoppedOrDisabled", e.Station, IsStationNotStoppedOrDisabled));
// Try to set the station's error and quit if it already had one set.
activityMachine.AddActivity(new ActOnResultActivity <bool>("SetError", () => e.Station.SetError(e), false, () => activityMachine.Quit("SetError returned false")));
// if tray present notify of a process anomaly occurred.
if (activityMachine.Tray != null)
{
activityMachine.UseRuntimeTrigger(new PropertyChangedTrigger("TrayStateChanged", activityMachine.Tray, "State"));
// Tell the machine to quit acting if the tray is lost.
activityMachine.SetQuitOnTrayState(TrayState.Lost);
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Notify Tray ProcessAnomaly {0}"
, trayProcessor.Name), () => activityMachine.Tray.OnProcessingAnomalyOccurred(e)));
// Pause here until the station is done with the current process and the tray has been retrieved.
activityMachine.AddContinueCondition(new DynamicConstraint <IStation>(trayProcessor.Name + " tray processing finished?", trayProcessor as IStation
, s => s.State == StationState.Idle && !(s as ITrayDetector).IsTrayDetected));
}
// Now get a resource lock for the error handling machine.
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Wait For Lock on {0}", trayProcessor.Name), () => trayProcessor.WaitForLock(activityMachine)));
activityMachine.AddQuitOrContinueCondition(new DynamicConstraint <ITrayProcessor>("StationState == StationState.Disabled", trayProcessor, s => s.State == StationState.Disabled), new DynamicConstraint <ITrayProcessor>("StationState != StationState.Disabled", trayProcessor, s => s.State != StationState.Disabled));
// Disable the station once processing is done and allow user to re-enable.
activityMachine.SetActivityDisable(AllowStationEnableAfterError);
// Make this machine wait until the station reports that it is indeed disabled by disable().
activityMachine.SetPauseUntilStationStatus(StationState.Disabled);
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Release Lock on {0}", trayProcessor.Name), () => trayProcessor.ReleaseLock(activityMachine)));
}
private void AssembleMachineForQuitOnErrorEscalation(DynamicActivityMachine machine, AtlasException e)
{
// This will trigger if either Error or Severity property notifies of a change.
machine.UseRuntimeTrigger(new PropertyChangedTrigger("Station Error Severity Changed", e.Station,
e.Station.PropertyToString(() => e.Station.Error) + "." + e.Station.PropertyToString(() => e.Station.Error.Severity)));
// Quit at any time if the station's error escalates in severity.
machine.UseAdditionalQuitCondition(new DynamicConstraint <IStation>("Station Error Severity Increased?", e.Station,
(s) => s.Error != null && s.Error.Severity.IsGreaterThan(e.Severity)));
}
protected override void SetSeverityActivities(DynamicActivityMachine machine, AtlasException e)
{
switch (e.Severity)
{
case ExceptionSeverityCategory.Fatal:
AssembleMachineForFatalError(machine as ErrorHandlingActivityMachine, machine.Configuration.Data.Instrument as IInstrument, e);
break;
case ExceptionSeverityCategory.Critical:
AssembleMachineForCriticalError(machine as ErrorHandlingActivityMachine, machine.Configuration.Data.Instrument as IInstrument, e);
break;
case ExceptionSeverityCategory.Severe:
AssembleMachineForSevereError(machine as ErrorHandlingActivityMachine, machine.Configuration.Data.Instrument as IInstrument, e);
break;
case ExceptionSeverityCategory.Standard:
AssembleMachineForStandardError(machine as ErrorHandlingActivityMachine, machine.Configuration.Data.Instrument as IInstrument, e);
break;
}
}
public void AssembleMachineForFatalError(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e)
{
// No resource locking required for fatal
//stop commands running on stations and shut down instrument.
SetStopInstrumentActivity(activityMachine, e);
}
public void AssembleMachineForReinitRecovery(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e)
{
var station = e.Station;
var trayDetector = station as ITrayDetector;
if (trayDetector != null)
{
activityMachine.UseRuntimeTrigger(new PropertyChangedTrigger("IsTrayDetected Changed", trayDetector,
trayDetector.PropertyToString(() => trayDetector.IsTrayDetected)));
}
activityMachine.AddActivity(new DynamicActivity("Log Reinitialize", () => LogService.Log(GetType().Name, "Begin reinitialize activity.")));
if (station is IReinitializable)
{
var reinitializableStation = station as IReinitializable;
if (reinitializableStation.ReinitAttempted)
{
activityMachine.AddActivity(new DynamicActivity("Log Reinitialize", () => LogService.Log(GetType().Name, "Station already attempted Reinit")));
return;
}
activityMachine.AddActivity(new DynamicActivity("Reinit set initial states", () => (e.Station as IReinitializable).Reinit(e)));
}
else
{
activityMachine.AddActivity(new DynamicActivity("Log Reinitialize", () => LogService.Log(GetType().Name, "Station is not IReinitializable")));
return;
}
AssembleMachineForAbortTray(activityMachine, instrument, e);
// Now get a resource lock for the remaining reinit operations.
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Wait For Lock on {0}", trayDetector.Name), () => trayDetector.WaitForLock(activityMachine)));
activityMachine.AddConditionalActivity(new DynamicConstraint <IStation>("Station has no tray and is idle.", e.Station, (s =>
(s is IReinitializable && (s as IReinitializable).AllowsReinitialization) && //1. Module is IReinitializable
((s is ITrayHandler && !(s as ITrayHandler).IsTrayDetected) || (s is ITrayProcessor && !(s as ITrayProcessor).IsTrayDetected)) && //2. Tray is not present
s.State == StationState.Idle)) //3. Module is idle
, new DynamicActivity("Station.Stop", () => e.Station.Stop()));
//this delays the start until the station state is stopped
activityMachine.AddConditionalActivity(new DynamicConstraint <IStation>("Station is stopped.", e.Station,
(s => s.State == StationState.Stopped))
, new DynamicActivity("Station.Reinitialize", () => e.Station.Start()));
activityMachine.AddContinueCondition(new DynamicConstraint <IStation>("Is Processing? (after Start)", e.Station,
(s => (!(e.Station is ITrayProcessor) || !(e.Station as ITrayProcessor).IsProcessing))));
activityMachine.AddActivity(new DynamicActivity("Station update Reinit State", () =>
{
var reinitializableStation = e.Station as IReinitializable;
if (reinitializableStation != null)
{
reinitializableStation.ReinitState = e.Station.State == StationState.Idle ? ReinitState.Success : ReinitState.Fail;
}
}));
activityMachine.AddFinishOrContinueCondition(
new DynamicConstraint <IStation>("Station reinitialize succeeded.", e.Station,
(s => s.State == StationState.Idle))
,
new DynamicConstraint <IStation>("Station reinitialize failed.", e.Station,
(s => s.State != StationState.Idle))
);
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Release Lock on {0}", trayDetector.Name), () => trayDetector.ReleaseLock(activityMachine)));
}
public void AssembleMachineForSevereError(ErrorHandlingActivityMachine activityMachine, IInstrument instrument, AtlasException e)
{
var trayProcessor = e.Station as ITrayProcessor;
AssembleMachineForQuitOnErrorEscalation(activityMachine, e);
SetDegradedModeActivity(activityMachine, e);
activityMachine.UseFinalExitBehavior(new DynamicActivity("Release locks", () => trayProcessor.ReleaseLock(activityMachine)));
// Only assemble a machine for an error if the station is running.
activityMachine.AddFinishOrContinueCondition(
new DynamicConstraint <IStation>("StationStoppedOrDisabled", e.Station, IsStationStoppedOrDisabled),
new DynamicConstraint <IStation>("StationNotStoppedOrDisabled", e.Station, IsStationNotStoppedOrDisabled));
// Try to set the station's error and quit if it already had one set.
activityMachine.AddActivity(new ActOnResultActivity <bool>("SetError", () => e.Station.SetError(e), false, () => activityMachine.Quit("SetError returned false")));
AssembleMachineForAbortTray(activityMachine, instrument, e);
// Now get a resource lock for the error handling machine.
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Wait For Lock on {0}", trayProcessor.Name),
() => trayProcessor.WaitForLock(activityMachine)));
activityMachine.AddQuitOrContinueCondition(new DynamicConstraint <ITrayProcessor>("StationState == StationState.Disabled",
trayProcessor, s => s.State == StationState.Disabled), new DynamicConstraint <ITrayProcessor>("StationState != StationState.Disabled",
trayProcessor, s => s.State != StationState.Disabled));
// Disable the station when it is finished processing.
activityMachine.SetActivityDisable(AllowStationEnableAfterError);
activityMachine.AddActivity(new DynamicActivity(string.Format(CultureInfo.InvariantCulture, "Release Lock on {0}", trayProcessor.Name),
() => trayProcessor.ReleaseLock(activityMachine)));
}
