private void LoadScanDetails(XRawFileIO xcaliburAccessor)
{
OnStatusEvent(" Loading scan details");
if (mSaveTICAndBPI)
{
// Initialize the TIC and BPI arrays
InitializeTICAndBPI();
}
if (mSaveLCMS2DPlots)
{
InitializeLCMS2DPlot();
}
var lastProgressTime = DateTime.UtcNow;
// Note that this starts at 2 seconds, but is extended after each progress message is shown (maxing out at 30 seconds)
var progressThresholdSeconds = 2;
var scanCount = xcaliburAccessor.GetNumScans();
GetStartAndEndScans(scanCount, out var scanStart, out var scanEnd);
var scansProcessed = 0;
var totalScansToProcess = scanEnd - scanStart + 1;
for (var scanNumber = scanStart; scanNumber <= scanEnd; scanNumber++)
{
clsScanInfo scanInfo;
try
{
var success = xcaliburAccessor.GetScanInfo(scanNumber, out scanInfo);
if (success)
{
if (mSaveTICAndBPI)
{
mTICAndBPIPlot.AddData(
scanNumber,
scanInfo.MSLevel,
(float)scanInfo.RetentionTime,
scanInfo.BasePeakIntensity,
scanInfo.TotalIonCurrent);
}
var scanStatsEntry = new ScanStatsEntry
{
ScanNumber = scanNumber,
ScanType = scanInfo.MSLevel,
ScanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(scanInfo.FilterText),
ScanFilterText = XRawFileIO.MakeGenericThermoScanFilter(scanInfo.FilterText),
ElutionTime = scanInfo.RetentionTime.ToString("0.0###"),
TotalIonIntensity = StringUtilities.ValueToString(scanInfo.TotalIonCurrent, 5),
BasePeakIntensity = StringUtilities.ValueToString(scanInfo.BasePeakIntensity, 5),
BasePeakMZ = scanInfo.BasePeakMZ.ToString("0.0###"),
BasePeakSignalToNoiseRatio = "0",
IonCount = scanInfo.NumPeaks,
IonCountRaw = scanInfo.NumPeaks,
MzMin = scanInfo.LowMass,
MzMax = scanInfo.HighMass
};
// Store the ScanEvent values in .ExtendedScanInfo
StoreExtendedScanInfo(scanStatsEntry.ExtendedScanInfo, scanInfo.ScanEvents);
// Store the collision mode and the scan filter text
scanStatsEntry.ExtendedScanInfo.CollisionMode = scanInfo.CollisionMode;
scanStatsEntry.ExtendedScanInfo.ScanFilterText = scanInfo.FilterText;
mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry);
}
}
catch (Exception ex)
{
OnErrorEvent("Error loading header info for scan " + scanNumber + ": " + ex.Message);
continue;
}
try
{
if (mSaveLCMS2DPlots || mCheckCentroidingStatus)
{
// Also need to load the raw data
// Load the ions for this scan
var ionCount = xcaliburAccessor.GetScanData2D(scanNumber, out var massIntensityPairs);
if (ionCount > 0)
{
if (mSaveLCMS2DPlots)
{
mLCMS2DPlot.AddScan2D(scanNumber, scanInfo.MSLevel, (float)scanInfo.RetentionTime, ionCount, massIntensityPairs);
}
if (mCheckCentroidingStatus)
{
var mzCount = massIntensityPairs.GetLength(1);
var mzList = new List <double>(mzCount);
for (var i = 0; i <= mzCount - 1; i++)
{
mzList.Add(massIntensityPairs[0, i]);
}
var centroidingStatus = GetCentroidStatus(scanNumber, scanInfo);
mDatasetStatsSummarizer.ClassifySpectrum(mzList, scanInfo.MSLevel, centroidingStatus, "Scan " + scanNumber);
}
}
}
}
catch (Exception ex)
{
OnErrorEvent("Error loading m/z and intensity values for scan " + scanNumber + ": " + ex.Message);
}
scansProcessed++;
if (DateTime.UtcNow.Subtract(lastProgressTime).TotalSeconds < progressThresholdSeconds)
{
continue;
}
lastProgressTime = DateTime.UtcNow;
if (progressThresholdSeconds < 30)
{
progressThresholdSeconds += 2;
}
var percentComplete = scansProcessed / (float)totalScansToProcess * 100;
OnProgressUpdate(string.Format("Spectra processed: {0:N0}", scansProcessed), percentComplete);
}
Console.WriteLine();
}
private void LoadFrameDetails(
DataReader uimfReader,
IReadOnlyDictionary <int, UIMFData.FrameType> dctMasterFrameList,
IReadOnlyList <int> masterFrameNumList)
{
const int BAD_TIC_OR_BPI = int.MinValue;
// The StartTime value for each frame is the number of minutes since 12:00 am
// If acquiring data from 11:59 pm through 12:00 am, the StartTime will reset to zero
clsTICandBPIPlotter pressurePlot;
if (mSaveTICAndBPI)
{
// Initialize the TIC and BPI arrays
InitializeTICAndBPI();
mTICAndBPIPlot.BPIXAxisLabel = "Frame number";
mTICAndBPIPlot.TICXAxisLabel = "Frame number";
if (mInstrumentSpecificPlots.Count == 0)
{
AddInstrumentSpecificPlot("Drift tube Pressure");
}
// Track drift tube pressures using mTIC in pressurePlot
pressurePlot = mInstrumentSpecificPlots.First();
pressurePlot.DeviceType = Device.Analog;
pressurePlot.BPIXAxisLabel = "Frame number";
pressurePlot.TICXAxisLabel = "Frame number";
pressurePlot.TICYAxisLabel = "Pressure";
pressurePlot.TICYAxisExponentialNotation = false;
pressurePlot.TICPlotAbbrev = "Pressure";
pressurePlot.TICAutoMinMaxY = true;
pressurePlot.RemoveZeroesFromEnds = true;
}
else
{
pressurePlot = new clsTICandBPIPlotter();
}
if (mSaveLCMS2DPlots)
{
InitializeLCMS2DPlot();
}
var lastProgressTime = DateTime.UtcNow;
// Note that this starts at 2 seconds, but is extended after each progress message is shown (maxing out at 30 seconds)
var progressThresholdSeconds = 2;
var globalParams = uimfReader.GetGlobalParams();
var globalMaxBins = globalParams.Bins;
var mzList = new double[globalMaxBins + 1];
var intensityList = new int[globalMaxBins + 1];
var ionsIntensity = new double[globalMaxBins + 1];
// Call .GetStartAndEndScans to get the start and end Frames
GetStartAndEndScans(globalParams.NumFrames, out var frameStart, out var frameEnd);
// Construct the TIC and BPI (of all frames)
ConstructTICAndBPI(uimfReader, frameStart, frameEnd, out var dctTIC, out var dctBPI);
Console.Write(" Loading frame details");
// Initialize the frame StartTime variables
double frameStartTimeInitial = -1;
double frameStartTimeAddon = 0;
double frameStartTimePrevious = -1;
double frameStartTimeCurrent = 0;
for (var masterFrameNumIndex = 0; masterFrameNumIndex <= masterFrameNumList.Count - 1; masterFrameNumIndex++)
{
var frameNumber = masterFrameNumList[masterFrameNumIndex];
if (!dctMasterFrameList.TryGetValue(frameNumber, out var eFrameType))
{
OnWarningEvent(string.Format(
"Frametype {0} not found in dictionary dctMasterFrameList; ignoring frame {1} in LoadFrameDetails",
eFrameType, frameNumber));
continue;
}
// Check whether the frame number is within the desired range
if (frameNumber < frameStart || frameNumber > frameEnd)
{
continue;
}
try
{
FrameParams frameParams;
try
{
frameParams = uimfReader.GetFrameParams(frameNumber);
}
catch (Exception)
{
Console.WriteLine("Exception obtaining frame parameters for frame " + frameNumber + "; will skip this frame");
frameParams = null;
}
if (frameParams == null || eFrameType == UIMFData.FrameType.Calibration)
{
continue;
}
var nonZeroPointsInFrame = uimfReader.GetCountPerFrame(frameNumber);
int msLevel;
if (frameParams.FrameType == UIMFData.FrameType.MS2)
{
msLevel = 2;
}
else
{
msLevel = 1;
}
// Read the frame StartTime
// This will be zero in older .UIMF files, or in files converted from Agilent .D directories
// In newer files, it is the number of minutes since 12:00 am
frameStartTimeCurrent = frameParams.GetValueDouble(FrameParamKeyType.StartTimeMinutes);
if (masterFrameNumIndex == 0 || frameStartTimeInitial < -0.9)
{
frameStartTimeInitial = frameStartTimeCurrent;
}
if (frameStartTimePrevious > 1400 && frameStartTimePrevious > frameStartTimeCurrent)
{
// We likely rolled over midnight; bump up frameStartTimeAddon by 1440 minutes
frameStartTimeAddon += 60 * 24;
}
// Compute the elution time (in minutes) of this frame
var elutionTime = frameStartTimeCurrent + frameStartTimeAddon - frameStartTimeInitial;
if (!dctBPI.TryGetValue(frameNumber, out var bpi))
{
bpi = BAD_TIC_OR_BPI;
}
if (!dctTIC.TryGetValue(frameNumber, out var tic))
{
tic = BAD_TIC_OR_BPI;
}
if (mSaveTICAndBPI)
{
if (bpi > BAD_TIC_OR_BPI && tic > BAD_TIC_OR_BPI)
{
mTICAndBPIPlot.AddData(frameNumber, msLevel, (float)elutionTime, bpi, tic);
}
var pressure = frameParams.GetValueDouble(FrameParamKeyType.PressureBack);
if (Math.Abs(pressure) < float.Epsilon)
{
pressure = frameParams.GetValueDouble(FrameParamKeyType.RearIonFunnelPressure);
}
if (Math.Abs(pressure) < float.Epsilon)
{
pressure = frameParams.GetValueDouble(FrameParamKeyType.IonFunnelTrapPressure);
}
if (Math.Abs(pressure) < float.Epsilon)
{
pressure = frameParams.GetValueDouble(FrameParamKeyType.PressureFront);
}
pressurePlot.AddDataTICOnly(frameNumber, msLevel, (float)elutionTime, pressure);
}
var scanStatsEntry = new ScanStatsEntry
{
ScanNumber = frameNumber,
ScanType = msLevel
};
if (msLevel <= 1)
{
scanStatsEntry.ScanTypeName = "HMS";
}
else
{
scanStatsEntry.ScanTypeName = "HMSn";
}
scanStatsEntry.ScanFilterText = "IMS";
scanStatsEntry.ExtendedScanInfo.ScanFilterText = scanStatsEntry.ScanFilterText;
scanStatsEntry.ElutionTime = elutionTime.ToString("0.0###");
if (tic > BAD_TIC_OR_BPI)
{
scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(tic, 5);
}
else
{
scanStatsEntry.TotalIonIntensity = "0";
}
if (bpi > BAD_TIC_OR_BPI)
{
scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(bpi, 5);
}
else
{
scanStatsEntry.BasePeakIntensity = "0";
}
scanStatsEntry.BasePeakMZ = "0";
// Base peak signal to noise ratio
scanStatsEntry.BasePeakSignalToNoiseRatio = "0";
scanStatsEntry.IonCount = nonZeroPointsInFrame;
scanStatsEntry.IonCountRaw = nonZeroPointsInFrame;
// For UIMF files, we only store one entry per frame
// Use the median drift time as the representative drift time
var medianScanNum = frameParams.Scans / 2;
var driftTimeMsec = uimfReader.GetDriftTime(frameNumber, medianScanNum, true);
scanStatsEntry.DriftTimeMsec = driftTimeMsec.ToString("0.0###");
mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry);
if (mSaveLCMS2DPlots || mCheckCentroidingStatus)
{
try
{
// Also need to load the raw data
// We have to clear the m/z and intensity arrays before calling GetSpectrum
Array.Clear(mzList, 0, mzList.Length);
Array.Clear(intensityList, 0, intensityList.Length);
// Process all of the IMS scans in this Frame to compute a summed spectrum representative of the frame
// In UIMF files from IMS04, if Frame_Parameters.Scans = 360, Frame_Scans will have scans 0 through 359
// In UIMF files from IMS08, prior to December 1, 2014, if Frame_Parameters.Scans = 374, Frame_Scans will have scans 0 through 373
// in UIMF files from IMS08, after December 1, 2014 if Frame_Parameters.Scans = 374, Frame_Scans will have scans 1 through 374
var ionCount = uimfReader.GetSpectrum(frameNumber, frameNumber, eFrameType, 0, frameParams.Scans, out mzList, out intensityList);
if (ionCount > 0)
{
// The m/z and intensity arrays might contain entries with m/z values of 0;
// need to copy the data in place to get the data in the correct format.
// In addition, we'll copy the intensity values from intensityList() into ionsIntensity()
if (ionCount > mzList.Length)
{
ionCount = mzList.Length;
}
if (ionsIntensity.Length < ionCount)
{
Array.Resize(ref ionsIntensity, ionCount);
}
var targetIndex = 0;
for (var ionIndex = 0; ionIndex <= ionCount - 1; ionIndex++)
{
if (mzList[ionIndex] > 0)
{
mzList[targetIndex] = mzList[ionIndex];
ionsIntensity[targetIndex] = intensityList[ionIndex];
targetIndex += 1;
}
}
ionCount = targetIndex;
if (ionCount > 0)
{
if (ionsIntensity.Length > ionCount)
{
Array.Resize(ref ionsIntensity, ionCount);
}
if (mSaveLCMS2DPlots)
{
mLCMS2DPlot.AddScan(frameNumber, msLevel, (float)elutionTime, ionCount, mzList, ionsIntensity);
}
if (mCheckCentroidingStatus)
{
mDatasetStatsSummarizer.ClassifySpectrum(ionCount, mzList, msLevel, "Frame " + frameNumber);
}
}
}
}
catch (Exception ex)
{
OnWarningEvent("Error loading m/z and intensity values for frame " + frameNumber + ": " + ex.Message);
}
}
}
catch (Exception ex)
{
OnWarningEvent("Error loading header info for frame " + frameNumber + ": " + ex.Message);
}
frameStartTimePrevious = frameStartTimeCurrent;
if (DateTime.UtcNow.Subtract(lastProgressTime).TotalSeconds < progressThresholdSeconds)
{
continue;
}
lastProgressTime = DateTime.UtcNow;
if (progressThresholdSeconds < 30)
{
progressThresholdSeconds += 2;
}
var percentComplete = masterFrameNumIndex / (float)masterFrameNumList.Count * 100;
OnProgressUpdate(string.Format("Frames processed: {0:N0}", masterFrameNumIndex), percentComplete);
}
Console.WriteLine();
}
private bool ProcessChemstationMSDataFile(string datafilePath, DatasetFileInfo datasetFileInfo)
{
bool success;
var currentIndex = 0;
try
{
using (var reader = new ChemstationMSFileReader.clsChemstationDataMSFileReader(datafilePath))
{
datasetFileInfo.AcqTimeStart = reader.Header.AcqDate;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(reader.Header.RetentionTimeMinutesEnd);
datasetFileInfo.ScanCount = reader.Header.SpectraCount;
for (var spectrumIndex = 0; spectrumIndex <= datasetFileInfo.ScanCount - 1; spectrumIndex++)
{
currentIndex = spectrumIndex;
ChemstationMSFileReader.clsSpectralRecord spectrum = null;
List <float> mzList = null;
List <int> intensityList = null;
const int msLevel = 1;
bool validSpectrum;
try
{
reader.GetSpectrum(spectrumIndex, ref spectrum);
mzList = spectrum.Mzs;
intensityList = spectrum.Intensities;
validSpectrum = true;
}
catch (Exception ex)
{
OnWarningEvent("Exception obtaining data from the MS file for spectrum index " + currentIndex + ": " + ex.Message);
validSpectrum = false;
}
if (validSpectrum)
{
var scanStatsEntry = new ScanStatsEntry
{
ScanNumber = spectrumIndex + 1,
ScanType = msLevel,
ScanTypeName = "GC-MS",
ScanFilterText = string.Empty,
ElutionTime = spectrum.RetentionTimeMinutes.ToString("0.0###"),
TotalIonIntensity = StringUtilities.ValueToString(spectrum.TIC, 1),
BasePeakIntensity = StringUtilities.ValueToString(spectrum.BasePeakAbundance, 1),
BasePeakMZ = spectrum.BasePeakMZ.ToString("0.0###"),
BasePeakSignalToNoiseRatio = "0",
IonCount = mzList.Count
};
scanStatsEntry.IonCountRaw = scanStatsEntry.IonCount;
mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry);
if (mSaveTICAndBPI)
{
mTICAndBPIPlot.AddData(scanStatsEntry.ScanNumber, msLevel, spectrum.RetentionTimeMinutes, spectrum.BasePeakAbundance, spectrum.TIC);
if (mzList.Count > 0)
{
var ionsMZ = new double[mzList.Count];
var ionsIntensity = new double[mzList.Count];
for (var index = 0; index <= mzList.Count - 1; index++)
{
ionsMZ[index] = mzList[index];
ionsIntensity[index] = intensityList[index];
}
mLCMS2DPlot.AddScan(scanStatsEntry.ScanNumber, msLevel, spectrum.RetentionTimeMinutes, ionsMZ.Length, ionsMZ, ionsIntensity);
}
}
if (mCheckCentroidingStatus)
{
var mzDoubles = new List <double>(mzList.Count);
mzDoubles.AddRange(mzList.Select(ion => (double)ion));
mDatasetStatsSummarizer.ClassifySpectrum(mzDoubles, msLevel, "Scan " + scanStatsEntry.ScanNumber);
}
}
}
}
success = true;
}
catch (Exception ex)
{
// Exception reading file
OnWarningEvent("Exception reading data from the MS file at spectrum index " + currentIndex + ": " + ex.Message);
success = false;
}
return(success);
}
public bool StoreMSSpectraInfo(
bool ticStored,
ref double runtimeMinutes,
bool skipExistingScans,
bool skipScansWithNoIons,
int maxScansToTrackInDetail,
int maxScansForTicAndBpi,
out int scanCountSuccess,
out int scanCountError)
{
scanCountSuccess = 0;
scanCountError = 0;
try
{
Console.WriteLine();
OnStatusEvent("Obtaining scan times and MSLevels (this could take several minutes)");
mLastScanLoadingDebugProgressTime = DateTime.UtcNow;
mLastScanLoadingStatusProgressTime = DateTime.UtcNow;
mReportedTotalSpectraToExamine = false;
mCancellationToken = new CancellationTokenSource();
var scanTimes = new double[0];
var msLevels = new byte[0];
mGetScanTimesStartTime = DateTime.UtcNow;
mGetScanTimesMaxWaitTimeSeconds = 90;
mGetScanTimesAutoAborted = false;
var minScanIndexWithoutScanTimes = int.MaxValue;
try
{
mPWiz.GetScanTimesAndMsLevels(mCancellationToken.Token, out scanTimes, out msLevels, MonitorScanTimeLoadingProgress);
}
catch (OperationCanceledException)
{
// mCancellationToken.Cancel was called in MonitorScanTimeLoadingProgress
// Determine the scan index where GetScanTimesAndMsLevels exited the for loop
for (var scanIndex = 0; scanIndex <= scanTimes.Length - 1; scanIndex++)
{
if (msLevels[scanIndex] > 0)
{
continue;
}
minScanIndexWithoutScanTimes = scanIndex;
break;
}
if (!mGetScanTimesAutoAborted && minScanIndexWithoutScanTimes < int.MaxValue)
{
// Manually aborted; shrink the arrays to reflect the amount of data that was actually loaded
Array.Resize(ref scanTimes, minScanIndexWithoutScanTimes);
Array.Resize(ref msLevels, minScanIndexWithoutScanTimes);
}
}
var spectrumCount = scanTimes.Length;
// The scan times returned by .GetScanTimesAndMsLevels() are the acquisition time in seconds from the start of the analysis
// Convert these to minutes
for (var scanIndex = 0; scanIndex <= spectrumCount - 1; scanIndex++)
{
if (scanIndex >= minScanIndexWithoutScanTimes)
{
break;
}
var scanTimeMinutes = scanTimes[scanIndex] / 60.0;
scanTimes[scanIndex] = scanTimeMinutes;
}
Console.WriteLine();
OnStatusEvent("Reading spectra");
var lastDebugProgressTime = DateTime.UtcNow;
var lastStatusProgressTime = DateTime.UtcNow;
var skippedEmptyScans = 0;
var scanNumber = 0;
var scansStored = 0;
var ticAndBpiScansStored = 0;
var scanCountHMS = 0;
var scanCountHMSn = 0;
var scanCountMS = 0;
var scanCountMSn = 0;
for (var scanIndex = 0; scanIndex <= spectrumCount - 1; scanIndex++)
{
try
{
var computeTIC = true;
var computeBPI = true;
// Obtain the raw mass spectrum
var msDataSpectrum = mPWiz.GetSpectrum(scanIndex);
scanNumber = scanIndex + 1;
if (scanIndex >= minScanIndexWithoutScanTimes)
{
// msDataSpectrum.RetentionTime is already in minutes
scanTimes[scanIndex] = msDataSpectrum.RetentionTime ?? 0;
if (msDataSpectrum.Level >= byte.MinValue && msDataSpectrum.Level <= byte.MaxValue)
{
msLevels[scanIndex] = (byte)msDataSpectrum.Level;
}
}
var scanStatsEntry = new ScanStatsEntry
{
ScanNumber = scanNumber,
ScanType = msDataSpectrum.Level
};
if (msLevels[scanIndex] > 1)
{
if (HighResMS2)
{
scanStatsEntry.ScanTypeName = "HMSn";
scanCountHMSn++;
}
else
{
scanStatsEntry.ScanTypeName = "MSn";
scanCountMSn++;
}
}
else
{
if (HighResMS1)
{
scanStatsEntry.ScanTypeName = "HMS";
scanCountHMS++;
}
else
{
scanStatsEntry.ScanTypeName = "MS";
scanCountMS++;
}
}
var driftTimeMsec = msDataSpectrum.DriftTimeMsec ?? 0;
scanStatsEntry.ScanFilterText = driftTimeMsec > 0 ? "IMS" : string.Empty;
scanStatsEntry.ExtendedScanInfo.ScanFilterText = scanStatsEntry.ScanFilterText;
scanStatsEntry.DriftTimeMsec = driftTimeMsec.ToString("0.0###");
scanStatsEntry.ElutionTime = scanTimes[scanIndex].ToString("0.0###");
// Bump up runtimeMinutes if necessary
if (scanTimes[scanIndex] > runtimeMinutes)
{
runtimeMinutes = scanTimes[scanIndex];
}
var spectrum = mPWiz.GetSpectrumObject(scanIndex);
if (MSDataFileReader.TryGetCVParamDouble(spectrum.cvParams, pwiz.CLI.cv.CVID.MS_total_ion_current, out var tic))
{
// For timsTOF data, this is the TIC of the entire frame
scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(tic, 5);
computeTIC = false;
}
if (MSDataFileReader.TryGetCVParamDouble(spectrum.cvParams, pwiz.CLI.cv.CVID.MS_base_peak_intensity, out var bpi))
{
// For timsTOF data, this is the BPI of the entire frame
// Additionally, for timsTOF data, MS_base_peak_m_z is not defined
scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(bpi, 5);
if (MSDataFileReader.TryGetCVParamDouble(spectrum.scanList.scans[0].cvParams, pwiz.CLI.cv.CVID.MS_base_peak_m_z, out var basePeakMzFromCvParams))
{
scanStatsEntry.BasePeakMZ = StringUtilities.ValueToString(basePeakMzFromCvParams, 5);
computeBPI = false;
}
}
if (string.IsNullOrEmpty(scanStatsEntry.ScanFilterText))
{
if (spectrum?.scanList?.scans.Count > 0)
{
// Bruker timsTOF datasets will have CVParam "inverse reduced ion mobility" for IMS spectra; check for this
foreach (var scanItem in spectrum.scanList.scans)
{
if (MSDataFileReader.TryGetCVParam(scanItem.cvParams, pwiz.CLI.cv.CVID.MS_inverse_reduced_ion_mobility, out _))
{
scanStatsEntry.ScanFilterText = "IMS";
break;
}
}
}
}
// Base peak signal to noise ratio
scanStatsEntry.BasePeakSignalToNoiseRatio = "0";
scanStatsEntry.IonCount = msDataSpectrum.Mzs.Length;
scanStatsEntry.IonCountRaw = scanStatsEntry.IonCount;
if ((computeBPI || computeTIC) && scanStatsEntry.IonCount > 0)
{
// Step through the raw data to compute the BPI and TIC
var mzList = msDataSpectrum.Mzs;
var intensities = msDataSpectrum.Intensities;
tic = 0;
bpi = 0;
double basePeakMZ = 0;
for (var index = 0; index <= mzList.Length - 1; index++)
{
tic += intensities[index];
if (intensities[index] > bpi)
{
bpi = intensities[index];
basePeakMZ = mzList[index];
}
}
scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(tic, 5);
scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(bpi, 5);
scanStatsEntry.BasePeakMZ = StringUtilities.ValueToString(basePeakMZ, 5);
}
var addScan = !skipExistingScans || skipExistingScans && !mDatasetStatsSummarizer.HasScanNumber(scanNumber);
if (addScan)
{
if (skipScansWithNoIons && scanStatsEntry.IonCount == 0)
{
skippedEmptyScans++;
if (skippedEmptyScans < 25 ||
skippedEmptyScans < 100 && skippedEmptyScans % 10 == 0 ||
skippedEmptyScans < 1000 && skippedEmptyScans % 100 == 0 ||
skippedEmptyScans < 10000 && skippedEmptyScans % 1000 == 0 ||
skippedEmptyScans < 100000 && skippedEmptyScans % 10000 == 0 ||
skippedEmptyScans % 100000 == 0)
{
ConsoleMsgUtils.ShowDebug("Skipping scan {0:N0} since no ions; {1:N0} total skipped scans", scanNumber, skippedEmptyScans);
}
}
else
{
if (maxScansToTrackInDetail < 0 || scansStored < maxScansToTrackInDetail)
{
mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry);
scansStored += 1;
}
}
}
if (mSaveTICAndBPI && !ticStored && (maxScansForTicAndBpi < 0 || ticAndBpiScansStored < maxScansForTicAndBpi))
{
mTICAndBPIPlot.AddData(scanStatsEntry.ScanNumber, msLevels[scanIndex], (float)scanTimes[scanIndex], bpi, tic);
ticAndBpiScansStored += 1;
}
if (mSaveLCMS2DPlots && addScan)
{
mLCMS2DPlot.AddScan(scanStatsEntry.ScanNumber, msLevels[scanIndex], (float)scanTimes[scanIndex], msDataSpectrum.Mzs.Length, msDataSpectrum.Mzs, msDataSpectrum.Intensities);
}
if (mCheckCentroidingStatus)
{
mDatasetStatsSummarizer.ClassifySpectrum(msDataSpectrum.Mzs, msLevels[scanIndex], "Scan " + scanStatsEntry.ScanNumber);
}
scanCountSuccess += 1;
}
catch (Exception ex)
{
OnErrorEvent("Error loading header info for scan " + scanNumber, ex);
scanCountError += 1;
}
if (DateTime.UtcNow.Subtract(lastStatusProgressTime).TotalMinutes > 5)
{
OnStatusEvent(string.Format("Reading spectra, loaded {0:N0} / {1:N0} spectra; " +
"{2:N0} HMS spectra; {3:N0} HMSn spectra; " +
"{4:N0} MS spectra; {5:N0} MSn spectra; " +
"max elution time is {6:F2} minutes",
scanNumber, spectrumCount,
scanCountHMS, scanCountHMSn,
scanCountMS, scanCountMSn,
runtimeMinutes));
lastStatusProgressTime = DateTime.UtcNow;
lastDebugProgressTime = DateTime.UtcNow;
continue;
}
if (DateTime.UtcNow.Subtract(lastDebugProgressTime).TotalSeconds < 15)
{
continue;
}
lastDebugProgressTime = DateTime.UtcNow;
var percentComplete = scanNumber / (float)spectrumCount * 100;
var percentCompleteOverall = clsMSFileInfoProcessorBaseClass.ComputeIncrementalProgress(
PROGRESS_SCAN_TIMES_LOADED,
clsMSFileInfoProcessorBaseClass.PROGRESS_SPECTRA_LOADED,
percentComplete);
OnProgressUpdate(string.Format("Spectra processed: {0:N0}", scanNumber), percentCompleteOverall);
}
mDatasetStatsSummarizer.StoreScanTypeTotals(scanCountHMS, scanCountHMSn,
scanCountMS, scanCountMSn,
runtimeMinutes);
var scanCountTotal = scanCountSuccess + scanCountError;
if (scanCountTotal == 0)
{
return(false);
}
// Return True if at least 50% of the spectra were successfully read
return(scanCountSuccess >= scanCountTotal / 2.0);
}
catch (AccessViolationException)
{
// Attempted to read or write protected memory. This is often an indication that other memory is corrupt.
if (!mWarnedAccessViolationException)
{
OnWarningEvent("Error reading instrument data with ProteoWizard: Attempted to read or write protected memory. " +
"The instrument data file is likely corrupt.");
mWarnedAccessViolationException = true;
}
mDatasetStatsSummarizer.CreateEmptyScanStatsFiles = false;
return(false);
}
catch (Exception ex)
{
OnErrorEvent("Error reading instrument data with ProteoWizard: " + ex.Message, ex);
return(false);
}
}
private void ProcessSRM(
string chromatogramID,
float[] scanTimes,
float[] intensities,
List <float> ticScanTimes,
IReadOnlyList <int> ticScanNumbers,
ref double runtimeMinutes,
IDictionary <int, Dictionary <double, double> > dct2DDataParent,
IDictionary <int, Dictionary <double, double> > dct2DDataProduct,
IDictionary <int, float> dct2DDataScanTimes)
{
// Attempt to parse out the product m/z
var parentMZFound = ExtractQ1MZ(chromatogramID, out var parentMZ);
var productMZFound = ExtractQ3MZ(chromatogramID, out var productMZ);
var spectrumCount = scanTimes.Length;
for (var index = 0; index <= spectrumCount - 1; index++)
{
// Find the ScanNumber in the TIC nearest to scanTimes[index]
var indexMatch = FindNearestInList(ticScanTimes, scanTimes[index]);
var scanNumber = ticScanNumbers[indexMatch];
// Bump up runtimeMinutes if necessary
if (scanTimes[index] > runtimeMinutes)
{
runtimeMinutes = scanTimes[index];
}
var scanStatsEntry = new ScanStatsEntry
{
ScanNumber = scanNumber,
ScanType = 1,
ScanTypeName = "SRM",
ScanFilterText = StripExtraFromChromatogramID(chromatogramID),
ElutionTime = scanTimes[index].ToString("0.0###"),
TotalIonIntensity = intensities[index].ToString("0.0"),
BasePeakIntensity = intensities[index].ToString("0.0")
};
if (parentMZFound)
{
scanStatsEntry.BasePeakMZ = parentMZ.ToString("0.0###");
}
else if (productMZFound)
{
scanStatsEntry.BasePeakMZ = productMZ.ToString("0.0###");
}
else
{
scanStatsEntry.BasePeakMZ = "0";
}
// Base peak signal to noise ratio
scanStatsEntry.BasePeakSignalToNoiseRatio = "0";
scanStatsEntry.IonCount = 1;
scanStatsEntry.IonCountRaw = 1;
mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry);
if (mSaveLCMS2DPlots && intensities[index] > 0)
{
// Store the m/z and intensity values in dct2DDataParent and dct2DDataProduct
if (parentMZFound)
{
Store2DPlotDataPoint(dct2DDataParent, scanNumber, parentMZ, intensities[index]);
}
if (productMZFound)
{
Store2DPlotDataPoint(dct2DDataProduct, scanNumber, productMZ, intensities[index]);
}
if (!dct2DDataScanTimes.ContainsKey(scanNumber))
{
dct2DDataScanTimes[scanNumber] = scanTimes[index];
}
}
}
}
/// <summary>
/// Examines the subdirectories in the specified zip file
/// Determines the oldest and newest modified analysis.baf files (or apexAcquisition.method file if analysis.baf files are not found)
/// Presumes this is the AcqStartTime and AcqEndTime
/// </summary>
/// <param name="zipFile"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if at least one valid file is found; otherwise false</returns>
/// <remarks></remarks>
private void DetermineAcqStartEndTime(FileInfo zipFile, DatasetFileInfo datasetFileInfo)
{
var success = false;
try
{
// Bump up the file size
datasetFileInfo.FileSizeBytes += zipFile.Length;
var fileNamesToFind = new List <string> {
"analysis.baf",
"apexAcquisition.method",
"submethods.xml"
};
var zipFileReader = new Ionic.Zip.ZipFile(zipFile.FullName);
foreach (var fileNameToFind in fileNamesToFind)
{
using (var zipEntry = zipFileReader.GetEnumerator())
{
while (zipEntry.MoveNext())
{
if (zipEntry.Current == null)
{
continue;
}
if (zipEntry.Current.IsDirectory)
{
continue;
}
// Split the filename on the forward slash character
var nameParts = zipEntry.Current.FileName.Split('/');
if (nameParts.Length <= 0)
{
continue;
}
if (!string.Equals(nameParts[nameParts.Length - 1], fileNameToFind, StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (zipEntry.Current.LastModified < datasetFileInfo.AcqTimeStart)
{
datasetFileInfo.AcqTimeStart = zipEntry.Current.LastModified;
}
if (zipEntry.Current.LastModified > datasetFileInfo.AcqTimeEnd)
{
datasetFileInfo.AcqTimeEnd = zipEntry.Current.LastModified;
}
// Bump up the scan count
datasetFileInfo.ScanCount += 1;
// Add a Scan Stats entry
var scanStatsEntry = new ScanStatsEntry
{
ScanNumber = datasetFileInfo.ScanCount,
ScanType = 1,
ScanTypeName = "MALDI-HMS",
ScanFilterText = string.Empty,
ElutionTime = "0",
TotalIonIntensity = "0",
BasePeakIntensity = "0",
BasePeakMZ = "0",
BasePeakSignalToNoiseRatio = "0",
IonCount = 0,
IonCountRaw = 0
};
// Base peak signal to noise ratio
mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry);
success = true;
}
}
if (success)
{
break;
}
}
}
catch (Exception ex)
{
OnErrorEvent("Error in DetermineAcqStartEndTime: " + ex.Message);
}
}
protected void SaveScanStatEntry(
StreamWriter writer,
clsScanList.eScanTypeConstants eScanType,
clsScanInfo currentScan,
int datasetID)
{
const char TAB_DELIMITER = '\t';
var scanStatsEntry = new ScanStatsEntry()
{
ScanNumber = currentScan.ScanNumber
};
if (eScanType == clsScanList.eScanTypeConstants.SurveyScan)
{
scanStatsEntry.ScanType = 1;
scanStatsEntry.ScanTypeName = string.Copy(currentScan.ScanTypeName);
}
else
{
if (currentScan.FragScanInfo.MSLevel <= 1)
{
// This is a fragmentation scan, so it must have a scan type of at least 2
scanStatsEntry.ScanType = 2;
}
else
{
// .MSLevel is 2 or higher, record the actual MSLevel value
scanStatsEntry.ScanType = currentScan.FragScanInfo.MSLevel;
}
scanStatsEntry.ScanTypeName = string.Copy(currentScan.ScanTypeName);
}
scanStatsEntry.ScanFilterText = currentScan.ScanHeaderText;
scanStatsEntry.ElutionTime = currentScan.ScanTime.ToString("0.0000");
scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(currentScan.TotalIonIntensity, 5);
scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(currentScan.BasePeakIonIntensity, 5);
scanStatsEntry.BasePeakMZ = StringUtilities.DblToString(currentScan.BasePeakIonMZ, 4);
// Base peak signal to noise ratio
scanStatsEntry.BasePeakSignalToNoiseRatio = StringUtilities.ValueToString(MASICPeakFinder.clsMASICPeakFinder.ComputeSignalToNoise(currentScan.BasePeakIonIntensity, currentScan.BaselineNoiseStats.NoiseLevel), 4);
scanStatsEntry.IonCount = currentScan.IonCount;
scanStatsEntry.IonCountRaw = currentScan.IonCountRaw;
mScanTracking.ScanStats.Add(scanStatsEntry);
var dataColumns = new List <string>()
{
datasetID.ToString(), // Dataset ID
scanStatsEntry.ScanNumber.ToString(), // Scan number
scanStatsEntry.ElutionTime, // Scan time (minutes)
scanStatsEntry.ScanType.ToString(), // Scan type (1 for MS, 2 for MS2, etc.)
scanStatsEntry.TotalIonIntensity, // Total ion intensity
scanStatsEntry.BasePeakIntensity, // Base peak ion intensity
scanStatsEntry.BasePeakMZ, // Base peak ion m/z
scanStatsEntry.BasePeakSignalToNoiseRatio, // Base peak signal to noise ratio
scanStatsEntry.IonCount.ToString(), // Number of peaks (aka ions) in the spectrum
scanStatsEntry.IonCountRaw.ToString(), // Number of peaks (aka ions) in the spectrum prior to any filtering
scanStatsEntry.ScanTypeName // Scan type name
};
writer.WriteLine(string.Join(TAB_DELIMITER.ToString(), dataColumns));
}
private List <udtScansDataType> LoadScansFile(string scansFilePath)
{
const string FILTERED_SCANS_SUFFIX = "_filtered_scans.csv";
var dctColumnInfo = new Dictionary <string, int>
{
{ "type", -1 },
{ "bpi", -1 },
{ "bpi_mz", -1 },
{ "tic", -1 },
{ "num_peaks", -1 },
{ "num_deisotoped", -1 }
};
var colIndexScanOrFrameNum = -1;
var colIndexScanOrFrameTime = -1;
var scanData = new List <udtScansDataType>();
var rowNumber = 0;
var colIndexScanInfo = -1;
if (!File.Exists(scansFilePath) && scansFilePath.ToLower().EndsWith(FILTERED_SCANS_SUFFIX.ToLower()))
{
scansFilePath = scansFilePath.Substring(0, scansFilePath.Length - FILTERED_SCANS_SUFFIX.Length) + "_scans.csv";
}
if (!File.Exists(scansFilePath))
{
OnWarningEvent("Warning: _scans.csv file is missing; will plot vs. scan number instead of vs. elution time");
return(scanData);
}
Console.WriteLine(" Reading the _scans.csv file");
using (var reader = new StreamReader(new FileStream(scansFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
{
while (!reader.EndOfStream)
{
rowNumber += 1;
var dataLine = reader.ReadLine();
if (string.IsNullOrWhiteSpace(dataLine))
{
continue;
}
var dataColumns = dataLine.Split(',').ToList();
if (rowNumber == 1)
{
// Parse the header row
ParseColumnHeaders(dctColumnInfo, dataColumns, "_scans.csv");
colIndexScanOrFrameNum = GetScanOrFrameColIndex(dataColumns, "_scans.csv");
colIndexScanOrFrameTime = dataColumns.IndexOf("frame_time");
if (colIndexScanOrFrameTime < 0)
{
colIndexScanOrFrameTime = dataColumns.IndexOf("scan_time");
}
// The info column will have data of the form "FTMS + p NSI Full ms [400.00-2000.00]" for Thermo datasets
// For .mzXML files, this fill will simply have an integer (and thus isn't useful)
// It may not be present in older _scans.csv files and is thus optional
colIndexScanInfo = dataColumns.IndexOf("info");
continue;
}
try
{
var udtScanData = new udtScansDataType();
int.TryParse(dataColumns[colIndexScanOrFrameNum], out udtScanData.Scan);
float.TryParse(dataColumns[colIndexScanOrFrameTime], out udtScanData.ElutionTime);
int.TryParse(dataColumns[dctColumnInfo["type"]], out udtScanData.MSLevel);
double.TryParse(dataColumns[dctColumnInfo["bpi"]], out udtScanData.BasePeakIntensity);
double.TryParse(dataColumns[dctColumnInfo["bpi_mz"]], out udtScanData.BasePeakMZ);
double.TryParse(dataColumns[dctColumnInfo["tic"]], out udtScanData.TotalIonCurrent);
int.TryParse(dataColumns[dctColumnInfo["num_peaks"]], out udtScanData.NumPeaks);
int.TryParse(dataColumns[dctColumnInfo["num_deisotoped"]], out udtScanData.NumDeisotoped);
if (colIndexScanInfo > 0)
{
var infoText = dataColumns[colIndexScanInfo];
// Only store infoText in .FilterText if infoText is not simply an integer
if (!int.TryParse(infoText, out _))
{
udtScanData.FilterText = infoText;
}
}
scanData.Add(udtScanData);
if (mSaveTICAndBPI)
{
mTICAndBPIPlot.AddData(udtScanData.Scan, udtScanData.MSLevel, udtScanData.ElutionTime, udtScanData.BasePeakIntensity, udtScanData.TotalIonCurrent);
}
string scanTypeName;
if (string.IsNullOrWhiteSpace(udtScanData.FilterText))
{
udtScanData.FilterText = udtScanData.MSLevel > 1 ? "HMSn" : "HMS";
scanTypeName = udtScanData.FilterText;
}
else
{
scanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(udtScanData.FilterText);
}
var scanStatsEntry = new ScanStatsEntry
{
ScanNumber = udtScanData.Scan,
ScanType = udtScanData.MSLevel,
ScanTypeName = scanTypeName,
ScanFilterText = XRawFileIO.MakeGenericThermoScanFilter(udtScanData.FilterText),
ElutionTime = udtScanData.ElutionTime.ToString("0.0###"),
TotalIonIntensity = StringUtilities.ValueToString(udtScanData.TotalIonCurrent, 5),
BasePeakIntensity = StringUtilities.ValueToString(udtScanData.BasePeakIntensity, 5),
BasePeakMZ = udtScanData.BasePeakMZ.ToString("0.0###"),
BasePeakSignalToNoiseRatio = "0",
IonCount = udtScanData.NumDeisotoped,
IonCountRaw = udtScanData.NumPeaks,
ExtendedScanInfo =
{
CollisionMode = string.Empty,
ScanFilterText = udtScanData.FilterText
}
};
mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry);
}
catch (Exception ex)
{
OnWarningEvent("Warning: Ignoring scan " + dataColumns[dctColumnInfo["scan_num"]] + " since data conversion error: " + ex.Message);
}
}
}
return(scanData);
}
