/// <summary>
/// Executes an instrument bump test operation.
/// </summary>
/// <returns>The completed event for this bump test.</returns>
/// <exception cref="FailedBumpTestException">
/// If anything extraordinary happened during the bump test.
/// </exception>
protected void BumpInstrument()
{
// SGF 14-Jun-2011 INS-1732
_cumulativeBumpTestResponseTime = 0;
try
{
List <InstalledComponent> workingSensorList = GetWorkingSensorList();
// SGF 24-Aug-2011 INS-2314
// Create sensor gas response objects for each sensor in the instrument
foreach (InstalledComponent installedComponent in workingSensorList)
{
if (!(installedComponent.Component is Sensor)) // Skip non-sensors.
{
continue;
}
if (!installedComponent.Component.Enabled) // Skip disabled sensors.
{
continue;
}
Sensor sensor = (Sensor)installedComponent.Component;
SensorGasResponse sgr = new SensorGasResponse(sensor.Uid, DateTime.UtcNow);
sgr.GasConcentration = new GasConcentration(sensor.CalibrationGas, sensor.CalibrationGasConcentration);
sgr.GasDetected = sensor.GasDetected;
sgr.Type = GasResponseType.Bump;
if (!sensor.IsBumpEnabled(GasEndPoints)) // For sensors that are not bump-enabled set the status as Skipped
{
sgr.Status = Status.Skipped; //Suresh 19-APR-2012 INS-4537 (DEV)
}
// SGF (Suresh) 3-Oct-2012 INS-2709
// If this is a sensor-type specific bump test, and this sensor should be skipped during the bump
// test operation, then set the Status on the Sensor Gas Response for this sensor to Skipped.
if (ComponentCodes.Count != 0 && !ComponentCodes.Contains(sensor.Type.Code))
{
sgr.Status = Status.Skipped;
}
_returnEvent.GasResponses.Add(sgr);
}
BumpTestInstrumentParallel(); // Also known as "quick bump".
}
finally
{
if (Master.Instance.SwitchService.BadPumpTubingDetectedDuringBump)
{
Master.Instance.ConsoleService.UpdateState(ConsoleState.BumpStoppedCheckTubing);
}
else
{
Master.Instance.ConsoleService.UpdateState(ConsoleState.BumpingInstrument); // Clear the reference to a step in the bump testing process
}
}
} // end-BumpInstrument
} // end-CalibrateInstrumentSequential
/// <summary>
/// Calibrate the sensor. Operates on "_component" member variable (an InstalledComponent)
/// Note that this routine assumes that the parent class has already zeroed sensor.
/// </summary>
/// <param name="instComp">Component to be calibrated</param>
/// <param name="gasEndPoints">Gas End Points to be used</param>
/// <param name="isO2HighBumpFailed">Whether the CalibrateSensor is called as a part of O2 High Bump Failure</param>
/// <returns></returns>
internal SensorGasResponse CalibrateSensor(InstalledComponent instComp, List <GasEndPoint> gasEndPoints, bool isO2HighBumpFailed)
{
_component = instComp;
GasEndPoints = gasEndPoints;
Log.Assert(this._component != null);
bool alreadyFailed = false;
GasEndPoint gasEndPoint;
Sensor sensor = (Sensor)_component.Component;
double currentConcentration = sensor.CalibrationGasConcentration; // SGF 11-Oct-2010 INS-1189
DateTime durationStart = DateTime.UtcNow;
List <SensorCalibrationLimits> sensorCalibrationLimits = _testOnlySensorCalibrationLimits == null ? new SensorCalibrationLimitsDataAccess().FindAll() : _testOnlySensorCalibrationLimits;
// SGF 14-Jun-2011 INS-1732
SensorGasResponse response = _returnEvent.GetSensorGasResponseByUid(sensor.Uid);
// If CalibrateSensor was called as a part of O2 High Bump Failure, the reponse
// object will be null and has to be initialized for calibration operation.
if (isO2HighBumpFailed)
{
response = new SensorGasResponse(sensor.Uid, DateTime.UtcNow);
response.GasConcentration = new GasConcentration(sensor.CalibrationGas, currentConcentration);
response.GasDetected = sensor.GasDetected;
response.Type = GasResponseType.Calibrate;
}
if (response == null)
{
Log.Debug("CALIBRATION: Skipping sensor " + _component.Position + " due to missing sensor gas response object!");
return(null);
}
response.Position = _component.Position;
// Add in whatever cylinder was used for zeroing. zeroGasEndPoint
// will have been set by zerosensor(). Should never be null, but check just in case.
if (_zeroingUsedGasEndPoint != null)
{
response.UsedGasEndPoints.Add(_zeroingUsedGasEndPoint);
}
try
{
// See if Zeroing was successful. If not, set up response to indicate
// the failure and then we can leave. (The finally block below will handle
// the response as we leave this method)
if (!_instrumentController.GetSensorZeroingStatus(_component.Position))
{
Log.Debug(string.Format("CALIBRATION: SKIPPING SENSOR {0} ({1}) DUE TO FAILED ZEROING!", _component.Position, _component.Component.Uid));
// Setup status with our failed zeroing and last cal status.
response.Status = Status.ZeroFailed;
response.Time = DateTime.UtcNow;
response.Reading = 0.0;
return(response);
}
//Suresh 16-JAN-2012 INS-2480 - Begin
if (!_instrumentController.IsSensorCalibrationEnabled(_component))
{
Log.Debug("CALIBRATION: Calibration is disabled for sensor " + _component.Position + " (" + sensor.Type.Code + ")");
//In the above if condition we have already checked for zeroing status, therefore we verywell know
//when execution comes here then zeroing is passed for current calibration disabled sensor.
response.Status = Status.ZeroPassed;
response.Time = DateTime.UtcNow;
response.Reading = 0.0;
Log.Debug("CALIBRATION: SKIPPING SENSOR " + _component.Position + " (" + sensor.Type.Code + ")");
return(response);
}
//Suresh 16-JAN-2012 INS-2480 - End
// SGF 03-Nov-2010 Single Sensor Cal and Bump
if (ComponentCodes.Count != 0 && !ComponentCodes.Contains(sensor.Type.Code))
{
Log.Debug(string.Format("CALIBRATION: Skipping sensor {0} ({1}) not included in schedule's specified component list.", _component.Position, sensor.Type.Code));
// This sensor will not be calibrated. Indicate that zeroing passed.
response.Status = Status.ZeroPassed;
response.Time = DateTime.UtcNow;
response.Reading = 0.0;
return(response);
}
//INS-7282 - To determine if the sensor is expired based on the configured sensor age and setup date. Applicable only to Service Account
if (IsSensorExpiredForServiceAccount(sensor, response, sensorCalibrationLimits))
{
Log.Debug(string.Format("CALIBRATION: IsSensorExpiredForServiceAccount returned TRUE for {0} at position {1}.", sensor.Type.Code, _component.Position));
Log.Debug(string.Format("CALIBRATION: Marking {0} sensor at position {1} as {2}.", sensor.Type.Code, _component.Position, Status.Failed));
response.Status = Status.Failed;
response.Time = DateTime.UtcNow;
response.Reading = 0.0;
return(response);
}
if (_instrumentController.TestForInstrumentReset(response, "calibrating sensor, checked zeroing status") == true)
{
Log.Warning("CALIBRATION: ABORTED DUE TO INSTRUMENT RESET");
return(response);
}
Log.Debug("CALIBRATION: Zeroing of sensor " + _component.Position + " determined as successful.");
// Continue to calibrate, until there is known success, failure, or timeout
while (true)
{
// We'll fail to get a gas end point when we've tried and failed on
// every available bottle of appropriate gas.
try
{
gasEndPoint = GetSensorGasEndPoint(_component); // Get gas end point for calibrating this sensor.
}
catch
{
if (alreadyFailed)
{
return(response);
}
throw;
}
try
{
if (gasEndPoint == null) // There is no gas available.?
{
throw new CorrectCalibrationGasUnavailable(_returnEvent.DockedInstrument.SerialNumber);
}
// Purge between each passes when switching between attached cylinders during CAL to clear gases in line.
// INETQA-4189 RHP v7.6. Also make sure that this CYLINDER SWITCH PURGE does not happen for O2 Calibration initiated as part of Bump Test(02 sensor)
if (_usedGasEndPoint == null)
{
_usedGasEndPoint = (sensor.Type.Code == SensorCode.O2) ? null : gasEndPoint;
}
else if (_usedGasEndPoint != gasEndPoint && !isO2HighBumpFailed)
{
Log.Debug("CYLINDER SWITCH DETECTED : CLEAR GASES IN LINES BEFORE CALIBRATING NEXT SENSOR");
Stopwatch cylinderSwitchPurgeStopwatch = Log.TimingBegin("CALIBRATING - PURGE(CYLINDER-SWITCH)");
new InstrumentPurgeOperation(PurgeType.CylinderSwitch, _instrumentController, GasEndPoints, _returnEvent, new List <SensorGasResponse> {
response
}).Execute();
Log.TimingEnd("CALIBRATING - PURGE(CYLINDER-SWITCH)", cylinderSwitchPurgeStopwatch);
_usedGasEndPoint = gasEndPoint;
}
//Suresh 18-OCT-2011 INS-2293
// Indicate on the console which sensor is being calibrated
Master.Instance.ConsoleService.UpdateState(ConsoleState.CalibratingInstrument, Master.Instance.ConsoleService.GetSensorLabel(sensor.Type.Code));
// Guarantee that the correct calibration gas concentration is available.
double availableConcentration = _instrumentController.SetCalibrationGasConcentration(_component, gasEndPoint);
// If we didn't find anything with the gas.
if (availableConcentration == 0.0)
{
throw new CorrectCalibrationGasUnavailable(sensor.CalibrationGas.Code);
}
// Set the gas concentration.
response.GasConcentration.Concentration = sensor.CalibrationGasConcentration;
Log.Debug("Calibrating gas: " + sensor.CalibrationGas.Code + " conc: " + sensor.CalibrationGasConcentration);
// Determine the length of time to calibrate before timing out. We add a an extra
// timeout cushion so we don't want the DS to timeout before the instrument.
TimeSpan calTimeOut = _instrumentController.GetSensorCalibrationTimeout(_component.Position) + _timeOutCushion;
if (_instrumentController.TestForInstrumentReset(response, "calibrating sensor, getting calibration timeout") == true)
{
Log.Warning("CALIBRATION: ABORTED DUE TO INSTRUMENT RESET");
return(response);
}
// Do any preconditioning necessary.
Stopwatch stopwatch = Log.TimingBegin("CAL - PRECONDITION SENSOR");
TimeSpan preTime = _instrumentController.PreconditionSensor(_component, gasEndPoint, response);
Log.TimingEnd("CAL - PRECONDITION SENSOR", stopwatch);
if (preTime.TotalSeconds > 0) // will return zero if no precondition performed/needed.
{
response.UsedGasEndPoints.Add(new UsedGasEndPoint(gasEndPoint, CylinderUsage.Precondition, preTime));
}
if (!Master.Instance.ControllerWrapper.IsDocked()) // Did user undock instrument during preconditioning?
{
throw new InstrumentNotDockedException();
}
// SGF 14-Jun-2011 INS-1732
response.ReadingAfterPreconditioning = _instrumentController.GetSensorReading(_component.Position, sensor.Resolution);
response.TimeAfterPreconditioning = DateTime.UtcNow;
if (_instrumentController.TestForInstrumentReset(response, "calibrating sensor, sensor preconditioned") == true)
{
Log.Warning("CALIBRATION: ABORTED DUE TO INSTRUMENT RESET");
return(response);
}
// SGF Jan-13-2009 DSW-173
stopwatch = Log.TimingBegin("CAL - PAUSE GAS FLOW");
_instrumentController.PauseGasFlow(gasEndPoint, sensor, response);
Log.TimingEnd("CAL - PAUSE GAS FLOW", stopwatch);
if (_instrumentController.TestForInstrumentReset(response, "calibrating sensor, gas flow paused") == true)
{
Log.Warning("CALIBRATION: ABORTED DUE TO INSTRUMENT RESET");
return(response);
}
// Get the sensor's maximum reading.
double maximumReading = _instrumentController.GetSensorMaximumReading(_component.Position, sensor.Resolution);
response.BaseLine = _instrumentController.GetSensorBaseline(_component.Position);
response.ZeroOffset = _instrumentController.GetSensorZeroOffset(_component.Position, sensor.Resolution);
response.AccessoryPump = _instrumentController.AccessoryPump;
int calFlowRate = _instrumentController.GetSensorCalibrationFlowRate(_component);
_instrumentController.OpenGasEndPoint(gasEndPoint, calFlowRate);
DateTime startTime = DateTime.UtcNow;
int totalReadings = 0;
Log.Debug("CALIBRATION: BEGINNING CALIBRATION ON POSITION " + _component.Position + ", UID=" + _component.Component.Uid);
// Send the command to begin calibrating.
_instrumentController.BeginSensorCalibration(new int[] { _component.Position });
Log.Debug("CALIBRATION: Taking readings every " + _WAIT_INTERVAL + " msecs");
#region CALIBRATION LOOP
stopwatch = Log.TimingBegin("CAL - CALIBRATE SENSOR");
Status calibrationStatus = Status.InProgress;
bool instResetting = false;
bool?isCalibrating = false;
bool hasAborted = false;
Pump.IsBadPumpTubing = false;
do
{
TimeSpan calTime = DateTime.UtcNow - startTime;
if (calTime > calTimeOut)
{
Log.Debug("CALIBRATION: DS timing out calibration. Setting status to + " + Status.Failed);
calibrationStatus = Status.Failed;
break;
}
#if !TEST
Thread.Sleep(_WAIT_INTERVAL);
#endif
if (!Master.Instance.ControllerWrapper.IsDocked()) // watch out for user undocking during the long sleep interval.
{
break;
}
//If bad pump tubing is detected, throw flow failed exception
//which will further down be handled to report to iNet the situation
//and display appropriate error on LCD.
if (Master.Instance.PumpWrapper.IsBadPumpTubing())
{
string msg = "CALIBRATION: Bad pump tubing is detected. Stopping calibration.";
Log.Debug(msg);
throw new FlowFailedException(gasEndPoint);
}
// Get current reading.
response.Reading = _instrumentController.GetSensorCalibrationReading(_component.Position, sensor.Resolution);
response.Time = DateTime.UtcNow;
instResetting = _instrumentController.TestForInstrumentReset(response, "calibrating sensor, getting reading");
totalReadings++;
Log.Debug("CALIBRATION: (" + _component.Position + "), Reading #" + totalReadings + ": " + response.Reading + ", Span: " + response.FullSpanReserve);
// isCalibrating will be null if the instrument reset (InstrumentAborted)
isCalibrating = _instrumentController.IsSensorCalibrating(_component.Position);
hasAborted = isCalibrating == null ? true : false;
}while (isCalibrating == true &&
Master.Instance.PumpWrapper.GetOpenValvePosition() > 0 &&
instResetting == false);
Log.TimingEnd("CAL - CALIBRATE SENSOR", stopwatch);
#endregion CALIBRATION LOOP
// If we detect we're undocked, then assume that's why we broke out of above loop
// debug: Do we really need this check?
if (!Master.Instance.ControllerWrapper.IsDocked())
{
string msg = "CALIBRATION: Aborting on sensor " + _component.Position + " - Undocked instrument.";
Log.Debug(msg);
throw new InstrumentNotDockedException();
}
if (instResetting == false)
{
instResetting = _instrumentController.TestForInstrumentReset(response, "calibrating sensor, calibration finished");
}
bool flowFailed = Master.Instance.PumpWrapper.GetOpenValvePosition() <= 0;
TimeSpan elapsedTime = DateTime.UtcNow - startTime;
// Put info for the cylinder used during the bump into the Response object.
// (iNet needs to know)
response.UsedGasEndPoints.Add(new UsedGasEndPoint(gasEndPoint, CylinderUsage.Calibration, elapsedTime));
// If we detect flow failure, then assume that's why we broke out of above loop.
if (flowFailed)
{
// TODO - Abort calibration on instrument?
string msg = "CALIBRATION: Aborting on sensor " + _component.Position + " - Flow failed.";
Log.Debug(msg);
throw new FlowFailedException(gasEndPoint);
}
if (calibrationStatus == Status.Failed) // Timed out in above loop by IDS?
{
// TODO - Tell instrument to abort calibration?
Log.Debug("CALIBRATION: TIMED OUT by DS after " + elapsedTime.TotalSeconds + " seconds!");
}
else if (instResetting == true)
{
Log.Warning("CALIBRATION: ABORTED DUE TO INSTRUMENT RESET");
return(response);
}
else // Find out if instrument decided to pass or fail the calibration
{
calibrationStatus = _instrumentController.GetSensorCalibrationStatus(_component.Position) ? Status.Passed : Status.SpanFailed;
response.Time = DateTime.UtcNow;
if (_instrumentController.TestForInstrumentReset(response, "calibrating sensor, retrieving calibration status") == true)
{
Log.Warning("CALIBRATION: ABORTED DUE TO INSTRUMENT RESET");
return(response);
}
Log.Debug(string.Format("CALIBRATION: Instrument {0} sensor {1} after {2} seconds!",
calibrationStatus, _component.Position, elapsedTime.TotalSeconds));
// Get instrument's final span reading
response.Reading = _instrumentController.GetSensorSpanReserve(_component.Position);
response.SpanCoef = _instrumentController.GetSensorSpanCoeff(_component.Position);
response.Status = calibrationStatus;
// Checking for obviously screwed up sensor.
if (hasAborted)
{
// we already know the instrument reset
response.Status = Status.InstrumentAborted;
}
else if (response.Status == Status.Passed)
{
// last calibration time is only changed on the sensor if calibration passed
response.PostCal_LastCalibrationTime = _instrumentController.GetSensorLastCalibrationTime(response.Position);
// status should never be passed if span reserve is 0 or below
if (response.FullSpanReserve <= 0)
{
Log.Warning(string.Format("CALIBRATION: FullSpanReserve is {0} but status is {1}. DS overriding with a Failed status.", response.FullSpanReserve, response.Status));
response.Status = Status.Failed;
}
// last calibration time (pre-cal vs post-cal) should have changed
else if (response.WasCalibrationInstrumentAborted())
{
Log.Warning(string.Format("CALIBRATION: Status is {0}, but LastCalibrationTime did not change. DS overriding with an InstrumentAborted status.", response.Status));
response.Status = Status.InstrumentAborted; // A new response object will be created so most values are cleared before uploading to iNet.
}
//INS-7282 - Check the sensor span reserve and if the value is less than the configured threshold, set the sensor calibration as failed. Applicable only to repair Accounts.
if (IsSensorFailedForRepairAccount(sensor, response, Configuration.DockingStation.SpanReserveThreshold))
{
response.Status = Status.Failed;
}
}
}
Log.Debug("CALIBRATION: " + response.Status
+ " - FullSpanReserve=" + response.FullSpanReserve
+ " SpanReading=" + response.Reading
+ " max=" + maximumReading);
if (response.Status == Status.Passed)
{
return(response);
}
alreadyFailed = true;
}
finally
{
_instrumentController.CloseGasEndPoint(gasEndPoint);
}
} // end-while(true)
}
finally
{
// How long did this sensor take to calibrate?
TimeSpan duration = DateTime.UtcNow - durationStart;
response.Duration = Convert.ToInt32(duration.TotalSeconds);
// SGF 14-Jun-2011 INS-1732
_cumulativeCalTestResponseTime = _cumulativeCalTestResponseTime + response.Duration;
response.CumulativeResponseTime = _cumulativeCalTestResponseTime;
ResetTriedGasEndPoints();
string msg = string.Empty;
try
{
if (Master.Instance.ControllerWrapper.IsDocked())
{
msg = "SetCalibrationGasConcentration";
//// Make certain oxygen sensors are set to ambient air concentrations.
////
//// TODO - Prior to the calibration, the O2 sensor may have had a cal gas
//// concentration setting other than 20.9, so we probably shouldn't be blindly setting it back to 20.9.
//// This likely needs to be corrected as part of INS-1189 (which used to be DSW-156, which used to be DSZ-1305).
//// - JMP, 9/28/2009
////
//if ( sensor.CalibrationGas.Code == GasCode.O2 )
// // Guarantee that the correct calibration gas concentration is available.
// SetCalibrationGasConcentration( GetSensorZeroAir( _component, false ) );
// SGF 11-Oct-2010 INS-1189
// Restore the concentration that was present upon entry into this method
_instrumentController.SetCalibrationGasConcentration(_component, currentConcentration, true);
msg = "GetSensorBumpStatus";
//Suresh 22-Feb-2012 INS-2705
//After calibration is completed , we need to update sensor bump test status because in scheduler we have logic
//to force calibration based on sensor BumpTestStatus
sensor.BumpTestStatus = _instrumentController.GetSensorBumpStatus(_component.Position);
// INETQA-4178 RHP v7.6 Update the return event BumpTestStatus as this is used the eventprocessor to update switch service instrument
// this is required for the scheduler logic discussed above
Sensor installedSensor = (Sensor)_returnEvent.DockedInstrument.InstalledComponents.Find(ic => ic.Component.Uid == sensor.Uid).Component;
if (installedSensor != null)
{
installedSensor.BumpTestStatus = sensor.BumpTestStatus;
}
}
}
catch (Exception e)
{
Log.Error(msg, e);
}
}
} // end-CalibrateSensor()
