private void LoadScanDetails(XRawFileIO xcaliburAccessor) { OnStatusEvent(" Loading scan details"); if (mSaveTICAndBPI) { // Initialize the TIC and BPI arrays InitializeTICAndBPI(); } if (mSaveLCMS2DPlots) { InitializeLCMS2DPlot(); } var lastProgressTime = DateTime.UtcNow; // Note that this starts at 2 seconds, but is extended after each progress message is shown (maxing out at 30 seconds) var progressThresholdSeconds = 2; var scanCount = xcaliburAccessor.GetNumScans(); GetStartAndEndScans(scanCount, out var scanStart, out var scanEnd); var scansProcessed = 0; var totalScansToProcess = scanEnd - scanStart + 1; for (var scanNumber = scanStart; scanNumber <= scanEnd; scanNumber++) { clsScanInfo scanInfo; try { var success = xcaliburAccessor.GetScanInfo(scanNumber, out scanInfo); if (success) { if (mSaveTICAndBPI) { mTICAndBPIPlot.AddData( scanNumber, scanInfo.MSLevel, (float)scanInfo.RetentionTime, scanInfo.BasePeakIntensity, scanInfo.TotalIonCurrent); } var scanStatsEntry = new ScanStatsEntry { ScanNumber = scanNumber, ScanType = scanInfo.MSLevel, ScanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(scanInfo.FilterText), ScanFilterText = XRawFileIO.MakeGenericThermoScanFilter(scanInfo.FilterText), ElutionTime = scanInfo.RetentionTime.ToString("0.0###"), TotalIonIntensity = StringUtilities.ValueToString(scanInfo.TotalIonCurrent, 5), BasePeakIntensity = StringUtilities.ValueToString(scanInfo.BasePeakIntensity, 5), BasePeakMZ = scanInfo.BasePeakMZ.ToString("0.0###"), BasePeakSignalToNoiseRatio = "0", IonCount = scanInfo.NumPeaks, IonCountRaw = scanInfo.NumPeaks, MzMin = scanInfo.LowMass, MzMax = scanInfo.HighMass }; // Store the ScanEvent values in .ExtendedScanInfo StoreExtendedScanInfo(scanStatsEntry.ExtendedScanInfo, scanInfo.ScanEvents); // Store the collision mode and the scan filter text scanStatsEntry.ExtendedScanInfo.CollisionMode = scanInfo.CollisionMode; scanStatsEntry.ExtendedScanInfo.ScanFilterText = scanInfo.FilterText; mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry); } } catch (Exception ex) { OnErrorEvent("Error loading header info for scan " + scanNumber + ": " + ex.Message); continue; } try { if (mSaveLCMS2DPlots || mCheckCentroidingStatus) { // Also need to load the raw data // Load the ions for this scan var ionCount = xcaliburAccessor.GetScanData2D(scanNumber, out var massIntensityPairs); if (ionCount > 0) { if (mSaveLCMS2DPlots) { mLCMS2DPlot.AddScan2D(scanNumber, scanInfo.MSLevel, (float)scanInfo.RetentionTime, ionCount, massIntensityPairs); } if (mCheckCentroidingStatus) { var mzCount = massIntensityPairs.GetLength(1); var mzList = new List <double>(mzCount); for (var i = 0; i <= mzCount - 1; i++) { mzList.Add(massIntensityPairs[0, i]); } var centroidingStatus = GetCentroidStatus(scanNumber, scanInfo); mDatasetStatsSummarizer.ClassifySpectrum(mzList, scanInfo.MSLevel, centroidingStatus, "Scan " + scanNumber); } } } } catch (Exception ex) { OnErrorEvent("Error loading m/z and intensity values for scan " + scanNumber + ": " + ex.Message); } scansProcessed++; if (DateTime.UtcNow.Subtract(lastProgressTime).TotalSeconds < progressThresholdSeconds) { continue; } lastProgressTime = DateTime.UtcNow; if (progressThresholdSeconds < 30) { progressThresholdSeconds += 2; } var percentComplete = scansProcessed / (float)totalScansToProcess * 100; OnProgressUpdate(string.Format("Spectra processed: {0:N0}", scansProcessed), percentComplete); } Console.WriteLine(); }
private void LoadFrameDetails( DataReader uimfReader, IReadOnlyDictionary <int, UIMFData.FrameType> dctMasterFrameList, IReadOnlyList <int> masterFrameNumList) { const int BAD_TIC_OR_BPI = int.MinValue; // The StartTime value for each frame is the number of minutes since 12:00 am // If acquiring data from 11:59 pm through 12:00 am, the StartTime will reset to zero clsTICandBPIPlotter pressurePlot; if (mSaveTICAndBPI) { // Initialize the TIC and BPI arrays InitializeTICAndBPI(); mTICAndBPIPlot.BPIXAxisLabel = "Frame number"; mTICAndBPIPlot.TICXAxisLabel = "Frame number"; if (mInstrumentSpecificPlots.Count == 0) { AddInstrumentSpecificPlot("Drift tube Pressure"); } // Track drift tube pressures using mTIC in pressurePlot pressurePlot = mInstrumentSpecificPlots.First(); pressurePlot.DeviceType = Device.Analog; pressurePlot.BPIXAxisLabel = "Frame number"; pressurePlot.TICXAxisLabel = "Frame number"; pressurePlot.TICYAxisLabel = "Pressure"; pressurePlot.TICYAxisExponentialNotation = false; pressurePlot.TICPlotAbbrev = "Pressure"; pressurePlot.TICAutoMinMaxY = true; pressurePlot.RemoveZeroesFromEnds = true; } else { pressurePlot = new clsTICandBPIPlotter(); } if (mSaveLCMS2DPlots) { InitializeLCMS2DPlot(); } var lastProgressTime = DateTime.UtcNow; // Note that this starts at 2 seconds, but is extended after each progress message is shown (maxing out at 30 seconds) var progressThresholdSeconds = 2; var globalParams = uimfReader.GetGlobalParams(); var globalMaxBins = globalParams.Bins; var mzList = new double[globalMaxBins + 1]; var intensityList = new int[globalMaxBins + 1]; var ionsIntensity = new double[globalMaxBins + 1]; // Call .GetStartAndEndScans to get the start and end Frames GetStartAndEndScans(globalParams.NumFrames, out var frameStart, out var frameEnd); // Construct the TIC and BPI (of all frames) ConstructTICAndBPI(uimfReader, frameStart, frameEnd, out var dctTIC, out var dctBPI); Console.Write(" Loading frame details"); // Initialize the frame StartTime variables double frameStartTimeInitial = -1; double frameStartTimeAddon = 0; double frameStartTimePrevious = -1; double frameStartTimeCurrent = 0; for (var masterFrameNumIndex = 0; masterFrameNumIndex <= masterFrameNumList.Count - 1; masterFrameNumIndex++) { var frameNumber = masterFrameNumList[masterFrameNumIndex]; if (!dctMasterFrameList.TryGetValue(frameNumber, out var eFrameType)) { OnWarningEvent(string.Format( "Frametype {0} not found in dictionary dctMasterFrameList; ignoring frame {1} in LoadFrameDetails", eFrameType, frameNumber)); continue; } // Check whether the frame number is within the desired range if (frameNumber < frameStart || frameNumber > frameEnd) { continue; } try { FrameParams frameParams; try { frameParams = uimfReader.GetFrameParams(frameNumber); } catch (Exception) { Console.WriteLine("Exception obtaining frame parameters for frame " + frameNumber + "; will skip this frame"); frameParams = null; } if (frameParams == null || eFrameType == UIMFData.FrameType.Calibration) { continue; } var nonZeroPointsInFrame = uimfReader.GetCountPerFrame(frameNumber); int msLevel; if (frameParams.FrameType == UIMFData.FrameType.MS2) { msLevel = 2; } else { msLevel = 1; } // Read the frame StartTime // This will be zero in older .UIMF files, or in files converted from Agilent .D directories // In newer files, it is the number of minutes since 12:00 am frameStartTimeCurrent = frameParams.GetValueDouble(FrameParamKeyType.StartTimeMinutes); if (masterFrameNumIndex == 0 || frameStartTimeInitial < -0.9) { frameStartTimeInitial = frameStartTimeCurrent; } if (frameStartTimePrevious > 1400 && frameStartTimePrevious > frameStartTimeCurrent) { // We likely rolled over midnight; bump up frameStartTimeAddon by 1440 minutes frameStartTimeAddon += 60 * 24; } // Compute the elution time (in minutes) of this frame var elutionTime = frameStartTimeCurrent + frameStartTimeAddon - frameStartTimeInitial; if (!dctBPI.TryGetValue(frameNumber, out var bpi)) { bpi = BAD_TIC_OR_BPI; } if (!dctTIC.TryGetValue(frameNumber, out var tic)) { tic = BAD_TIC_OR_BPI; } if (mSaveTICAndBPI) { if (bpi > BAD_TIC_OR_BPI && tic > BAD_TIC_OR_BPI) { mTICAndBPIPlot.AddData(frameNumber, msLevel, (float)elutionTime, bpi, tic); } var pressure = frameParams.GetValueDouble(FrameParamKeyType.PressureBack); if (Math.Abs(pressure) < float.Epsilon) { pressure = frameParams.GetValueDouble(FrameParamKeyType.RearIonFunnelPressure); } if (Math.Abs(pressure) < float.Epsilon) { pressure = frameParams.GetValueDouble(FrameParamKeyType.IonFunnelTrapPressure); } if (Math.Abs(pressure) < float.Epsilon) { pressure = frameParams.GetValueDouble(FrameParamKeyType.PressureFront); } pressurePlot.AddDataTICOnly(frameNumber, msLevel, (float)elutionTime, pressure); } var scanStatsEntry = new ScanStatsEntry { ScanNumber = frameNumber, ScanType = msLevel }; if (msLevel <= 1) { scanStatsEntry.ScanTypeName = "HMS"; } else { scanStatsEntry.ScanTypeName = "HMSn"; } scanStatsEntry.ScanFilterText = "IMS"; scanStatsEntry.ExtendedScanInfo.ScanFilterText = scanStatsEntry.ScanFilterText; scanStatsEntry.ElutionTime = elutionTime.ToString("0.0###"); if (tic > BAD_TIC_OR_BPI) { scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(tic, 5); } else { scanStatsEntry.TotalIonIntensity = "0"; } if (bpi > BAD_TIC_OR_BPI) { scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(bpi, 5); } else { scanStatsEntry.BasePeakIntensity = "0"; } scanStatsEntry.BasePeakMZ = "0"; // Base peak signal to noise ratio scanStatsEntry.BasePeakSignalToNoiseRatio = "0"; scanStatsEntry.IonCount = nonZeroPointsInFrame; scanStatsEntry.IonCountRaw = nonZeroPointsInFrame; // For UIMF files, we only store one entry per frame // Use the median drift time as the representative drift time var medianScanNum = frameParams.Scans / 2; var driftTimeMsec = uimfReader.GetDriftTime(frameNumber, medianScanNum, true); scanStatsEntry.DriftTimeMsec = driftTimeMsec.ToString("0.0###"); mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry); if (mSaveLCMS2DPlots || mCheckCentroidingStatus) { try { // Also need to load the raw data // We have to clear the m/z and intensity arrays before calling GetSpectrum Array.Clear(mzList, 0, mzList.Length); Array.Clear(intensityList, 0, intensityList.Length); // Process all of the IMS scans in this Frame to compute a summed spectrum representative of the frame // In UIMF files from IMS04, if Frame_Parameters.Scans = 360, Frame_Scans will have scans 0 through 359 // In UIMF files from IMS08, prior to December 1, 2014, if Frame_Parameters.Scans = 374, Frame_Scans will have scans 0 through 373 // in UIMF files from IMS08, after December 1, 2014 if Frame_Parameters.Scans = 374, Frame_Scans will have scans 1 through 374 var ionCount = uimfReader.GetSpectrum(frameNumber, frameNumber, eFrameType, 0, frameParams.Scans, out mzList, out intensityList); if (ionCount > 0) { // The m/z and intensity arrays might contain entries with m/z values of 0; // need to copy the data in place to get the data in the correct format. // In addition, we'll copy the intensity values from intensityList() into ionsIntensity() if (ionCount > mzList.Length) { ionCount = mzList.Length; } if (ionsIntensity.Length < ionCount) { Array.Resize(ref ionsIntensity, ionCount); } var targetIndex = 0; for (var ionIndex = 0; ionIndex <= ionCount - 1; ionIndex++) { if (mzList[ionIndex] > 0) { mzList[targetIndex] = mzList[ionIndex]; ionsIntensity[targetIndex] = intensityList[ionIndex]; targetIndex += 1; } } ionCount = targetIndex; if (ionCount > 0) { if (ionsIntensity.Length > ionCount) { Array.Resize(ref ionsIntensity, ionCount); } if (mSaveLCMS2DPlots) { mLCMS2DPlot.AddScan(frameNumber, msLevel, (float)elutionTime, ionCount, mzList, ionsIntensity); } if (mCheckCentroidingStatus) { mDatasetStatsSummarizer.ClassifySpectrum(ionCount, mzList, msLevel, "Frame " + frameNumber); } } } } catch (Exception ex) { OnWarningEvent("Error loading m/z and intensity values for frame " + frameNumber + ": " + ex.Message); } } } catch (Exception ex) { OnWarningEvent("Error loading header info for frame " + frameNumber + ": " + ex.Message); } frameStartTimePrevious = frameStartTimeCurrent; if (DateTime.UtcNow.Subtract(lastProgressTime).TotalSeconds < progressThresholdSeconds) { continue; } lastProgressTime = DateTime.UtcNow; if (progressThresholdSeconds < 30) { progressThresholdSeconds += 2; } var percentComplete = masterFrameNumIndex / (float)masterFrameNumList.Count * 100; OnProgressUpdate(string.Format("Frames processed: {0:N0}", masterFrameNumIndex), percentComplete); } Console.WriteLine(); }
private bool ProcessChemstationMSDataFile(string datafilePath, DatasetFileInfo datasetFileInfo) { bool success; var currentIndex = 0; try { using (var reader = new ChemstationMSFileReader.clsChemstationDataMSFileReader(datafilePath)) { datasetFileInfo.AcqTimeStart = reader.Header.AcqDate; datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(reader.Header.RetentionTimeMinutesEnd); datasetFileInfo.ScanCount = reader.Header.SpectraCount; for (var spectrumIndex = 0; spectrumIndex <= datasetFileInfo.ScanCount - 1; spectrumIndex++) { currentIndex = spectrumIndex; ChemstationMSFileReader.clsSpectralRecord spectrum = null; List <float> mzList = null; List <int> intensityList = null; const int msLevel = 1; bool validSpectrum; try { reader.GetSpectrum(spectrumIndex, ref spectrum); mzList = spectrum.Mzs; intensityList = spectrum.Intensities; validSpectrum = true; } catch (Exception ex) { OnWarningEvent("Exception obtaining data from the MS file for spectrum index " + currentIndex + ": " + ex.Message); validSpectrum = false; } if (validSpectrum) { var scanStatsEntry = new ScanStatsEntry { ScanNumber = spectrumIndex + 1, ScanType = msLevel, ScanTypeName = "GC-MS", ScanFilterText = string.Empty, ElutionTime = spectrum.RetentionTimeMinutes.ToString("0.0###"), TotalIonIntensity = StringUtilities.ValueToString(spectrum.TIC, 1), BasePeakIntensity = StringUtilities.ValueToString(spectrum.BasePeakAbundance, 1), BasePeakMZ = spectrum.BasePeakMZ.ToString("0.0###"), BasePeakSignalToNoiseRatio = "0", IonCount = mzList.Count }; scanStatsEntry.IonCountRaw = scanStatsEntry.IonCount; mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry); if (mSaveTICAndBPI) { mTICAndBPIPlot.AddData(scanStatsEntry.ScanNumber, msLevel, spectrum.RetentionTimeMinutes, spectrum.BasePeakAbundance, spectrum.TIC); if (mzList.Count > 0) { var ionsMZ = new double[mzList.Count]; var ionsIntensity = new double[mzList.Count]; for (var index = 0; index <= mzList.Count - 1; index++) { ionsMZ[index] = mzList[index]; ionsIntensity[index] = intensityList[index]; } mLCMS2DPlot.AddScan(scanStatsEntry.ScanNumber, msLevel, spectrum.RetentionTimeMinutes, ionsMZ.Length, ionsMZ, ionsIntensity); } } if (mCheckCentroidingStatus) { var mzDoubles = new List <double>(mzList.Count); mzDoubles.AddRange(mzList.Select(ion => (double)ion)); mDatasetStatsSummarizer.ClassifySpectrum(mzDoubles, msLevel, "Scan " + scanStatsEntry.ScanNumber); } } } } success = true; } catch (Exception ex) { // Exception reading file OnWarningEvent("Exception reading data from the MS file at spectrum index " + currentIndex + ": " + ex.Message); success = false; } return(success); }
public bool StoreMSSpectraInfo( bool ticStored, ref double runtimeMinutes, bool skipExistingScans, bool skipScansWithNoIons, int maxScansToTrackInDetail, int maxScansForTicAndBpi, out int scanCountSuccess, out int scanCountError) { scanCountSuccess = 0; scanCountError = 0; try { Console.WriteLine(); OnStatusEvent("Obtaining scan times and MSLevels (this could take several minutes)"); mLastScanLoadingDebugProgressTime = DateTime.UtcNow; mLastScanLoadingStatusProgressTime = DateTime.UtcNow; mReportedTotalSpectraToExamine = false; mCancellationToken = new CancellationTokenSource(); var scanTimes = new double[0]; var msLevels = new byte[0]; mGetScanTimesStartTime = DateTime.UtcNow; mGetScanTimesMaxWaitTimeSeconds = 90; mGetScanTimesAutoAborted = false; var minScanIndexWithoutScanTimes = int.MaxValue; try { mPWiz.GetScanTimesAndMsLevels(mCancellationToken.Token, out scanTimes, out msLevels, MonitorScanTimeLoadingProgress); } catch (OperationCanceledException) { // mCancellationToken.Cancel was called in MonitorScanTimeLoadingProgress // Determine the scan index where GetScanTimesAndMsLevels exited the for loop for (var scanIndex = 0; scanIndex <= scanTimes.Length - 1; scanIndex++) { if (msLevels[scanIndex] > 0) { continue; } minScanIndexWithoutScanTimes = scanIndex; break; } if (!mGetScanTimesAutoAborted && minScanIndexWithoutScanTimes < int.MaxValue) { // Manually aborted; shrink the arrays to reflect the amount of data that was actually loaded Array.Resize(ref scanTimes, minScanIndexWithoutScanTimes); Array.Resize(ref msLevels, minScanIndexWithoutScanTimes); } } var spectrumCount = scanTimes.Length; // The scan times returned by .GetScanTimesAndMsLevels() are the acquisition time in seconds from the start of the analysis // Convert these to minutes for (var scanIndex = 0; scanIndex <= spectrumCount - 1; scanIndex++) { if (scanIndex >= minScanIndexWithoutScanTimes) { break; } var scanTimeMinutes = scanTimes[scanIndex] / 60.0; scanTimes[scanIndex] = scanTimeMinutes; } Console.WriteLine(); OnStatusEvent("Reading spectra"); var lastDebugProgressTime = DateTime.UtcNow; var lastStatusProgressTime = DateTime.UtcNow; var skippedEmptyScans = 0; var scanNumber = 0; var scansStored = 0; var ticAndBpiScansStored = 0; var scanCountHMS = 0; var scanCountHMSn = 0; var scanCountMS = 0; var scanCountMSn = 0; for (var scanIndex = 0; scanIndex <= spectrumCount - 1; scanIndex++) { try { var computeTIC = true; var computeBPI = true; // Obtain the raw mass spectrum var msDataSpectrum = mPWiz.GetSpectrum(scanIndex); scanNumber = scanIndex + 1; if (scanIndex >= minScanIndexWithoutScanTimes) { // msDataSpectrum.RetentionTime is already in minutes scanTimes[scanIndex] = msDataSpectrum.RetentionTime ?? 0; if (msDataSpectrum.Level >= byte.MinValue && msDataSpectrum.Level <= byte.MaxValue) { msLevels[scanIndex] = (byte)msDataSpectrum.Level; } } var scanStatsEntry = new ScanStatsEntry { ScanNumber = scanNumber, ScanType = msDataSpectrum.Level }; if (msLevels[scanIndex] > 1) { if (HighResMS2) { scanStatsEntry.ScanTypeName = "HMSn"; scanCountHMSn++; } else { scanStatsEntry.ScanTypeName = "MSn"; scanCountMSn++; } } else { if (HighResMS1) { scanStatsEntry.ScanTypeName = "HMS"; scanCountHMS++; } else { scanStatsEntry.ScanTypeName = "MS"; scanCountMS++; } } var driftTimeMsec = msDataSpectrum.DriftTimeMsec ?? 0; scanStatsEntry.ScanFilterText = driftTimeMsec > 0 ? "IMS" : string.Empty; scanStatsEntry.ExtendedScanInfo.ScanFilterText = scanStatsEntry.ScanFilterText; scanStatsEntry.DriftTimeMsec = driftTimeMsec.ToString("0.0###"); scanStatsEntry.ElutionTime = scanTimes[scanIndex].ToString("0.0###"); // Bump up runtimeMinutes if necessary if (scanTimes[scanIndex] > runtimeMinutes) { runtimeMinutes = scanTimes[scanIndex]; } var spectrum = mPWiz.GetSpectrumObject(scanIndex); if (MSDataFileReader.TryGetCVParamDouble(spectrum.cvParams, pwiz.CLI.cv.CVID.MS_total_ion_current, out var tic)) { // For timsTOF data, this is the TIC of the entire frame scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(tic, 5); computeTIC = false; } if (MSDataFileReader.TryGetCVParamDouble(spectrum.cvParams, pwiz.CLI.cv.CVID.MS_base_peak_intensity, out var bpi)) { // For timsTOF data, this is the BPI of the entire frame // Additionally, for timsTOF data, MS_base_peak_m_z is not defined scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(bpi, 5); if (MSDataFileReader.TryGetCVParamDouble(spectrum.scanList.scans[0].cvParams, pwiz.CLI.cv.CVID.MS_base_peak_m_z, out var basePeakMzFromCvParams)) { scanStatsEntry.BasePeakMZ = StringUtilities.ValueToString(basePeakMzFromCvParams, 5); computeBPI = false; } } if (string.IsNullOrEmpty(scanStatsEntry.ScanFilterText)) { if (spectrum?.scanList?.scans.Count > 0) { // Bruker timsTOF datasets will have CVParam "inverse reduced ion mobility" for IMS spectra; check for this foreach (var scanItem in spectrum.scanList.scans) { if (MSDataFileReader.TryGetCVParam(scanItem.cvParams, pwiz.CLI.cv.CVID.MS_inverse_reduced_ion_mobility, out _)) { scanStatsEntry.ScanFilterText = "IMS"; break; } } } } // Base peak signal to noise ratio scanStatsEntry.BasePeakSignalToNoiseRatio = "0"; scanStatsEntry.IonCount = msDataSpectrum.Mzs.Length; scanStatsEntry.IonCountRaw = scanStatsEntry.IonCount; if ((computeBPI || computeTIC) && scanStatsEntry.IonCount > 0) { // Step through the raw data to compute the BPI and TIC var mzList = msDataSpectrum.Mzs; var intensities = msDataSpectrum.Intensities; tic = 0; bpi = 0; double basePeakMZ = 0; for (var index = 0; index <= mzList.Length - 1; index++) { tic += intensities[index]; if (intensities[index] > bpi) { bpi = intensities[index]; basePeakMZ = mzList[index]; } } scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(tic, 5); scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(bpi, 5); scanStatsEntry.BasePeakMZ = StringUtilities.ValueToString(basePeakMZ, 5); } var addScan = !skipExistingScans || skipExistingScans && !mDatasetStatsSummarizer.HasScanNumber(scanNumber); if (addScan) { if (skipScansWithNoIons && scanStatsEntry.IonCount == 0) { skippedEmptyScans++; if (skippedEmptyScans < 25 || skippedEmptyScans < 100 && skippedEmptyScans % 10 == 0 || skippedEmptyScans < 1000 && skippedEmptyScans % 100 == 0 || skippedEmptyScans < 10000 && skippedEmptyScans % 1000 == 0 || skippedEmptyScans < 100000 && skippedEmptyScans % 10000 == 0 || skippedEmptyScans % 100000 == 0) { ConsoleMsgUtils.ShowDebug("Skipping scan {0:N0} since no ions; {1:N0} total skipped scans", scanNumber, skippedEmptyScans); } } else { if (maxScansToTrackInDetail < 0 || scansStored < maxScansToTrackInDetail) { mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry); scansStored += 1; } } } if (mSaveTICAndBPI && !ticStored && (maxScansForTicAndBpi < 0 || ticAndBpiScansStored < maxScansForTicAndBpi)) { mTICAndBPIPlot.AddData(scanStatsEntry.ScanNumber, msLevels[scanIndex], (float)scanTimes[scanIndex], bpi, tic); ticAndBpiScansStored += 1; } if (mSaveLCMS2DPlots && addScan) { mLCMS2DPlot.AddScan(scanStatsEntry.ScanNumber, msLevels[scanIndex], (float)scanTimes[scanIndex], msDataSpectrum.Mzs.Length, msDataSpectrum.Mzs, msDataSpectrum.Intensities); } if (mCheckCentroidingStatus) { mDatasetStatsSummarizer.ClassifySpectrum(msDataSpectrum.Mzs, msLevels[scanIndex], "Scan " + scanStatsEntry.ScanNumber); } scanCountSuccess += 1; } catch (Exception ex) { OnErrorEvent("Error loading header info for scan " + scanNumber, ex); scanCountError += 1; } if (DateTime.UtcNow.Subtract(lastStatusProgressTime).TotalMinutes > 5) { OnStatusEvent(string.Format("Reading spectra, loaded {0:N0} / {1:N0} spectra; " + "{2:N0} HMS spectra; {3:N0} HMSn spectra; " + "{4:N0} MS spectra; {5:N0} MSn spectra; " + "max elution time is {6:F2} minutes", scanNumber, spectrumCount, scanCountHMS, scanCountHMSn, scanCountMS, scanCountMSn, runtimeMinutes)); lastStatusProgressTime = DateTime.UtcNow; lastDebugProgressTime = DateTime.UtcNow; continue; } if (DateTime.UtcNow.Subtract(lastDebugProgressTime).TotalSeconds < 15) { continue; } lastDebugProgressTime = DateTime.UtcNow; var percentComplete = scanNumber / (float)spectrumCount * 100; var percentCompleteOverall = clsMSFileInfoProcessorBaseClass.ComputeIncrementalProgress( PROGRESS_SCAN_TIMES_LOADED, clsMSFileInfoProcessorBaseClass.PROGRESS_SPECTRA_LOADED, percentComplete); OnProgressUpdate(string.Format("Spectra processed: {0:N0}", scanNumber), percentCompleteOverall); } mDatasetStatsSummarizer.StoreScanTypeTotals(scanCountHMS, scanCountHMSn, scanCountMS, scanCountMSn, runtimeMinutes); var scanCountTotal = scanCountSuccess + scanCountError; if (scanCountTotal == 0) { return(false); } // Return True if at least 50% of the spectra were successfully read return(scanCountSuccess >= scanCountTotal / 2.0); } catch (AccessViolationException) { // Attempted to read or write protected memory. This is often an indication that other memory is corrupt. if (!mWarnedAccessViolationException) { OnWarningEvent("Error reading instrument data with ProteoWizard: Attempted to read or write protected memory. " + "The instrument data file is likely corrupt."); mWarnedAccessViolationException = true; } mDatasetStatsSummarizer.CreateEmptyScanStatsFiles = false; return(false); } catch (Exception ex) { OnErrorEvent("Error reading instrument data with ProteoWizard: " + ex.Message, ex); return(false); } }
private void ProcessSRM( string chromatogramID, float[] scanTimes, float[] intensities, List <float> ticScanTimes, IReadOnlyList <int> ticScanNumbers, ref double runtimeMinutes, IDictionary <int, Dictionary <double, double> > dct2DDataParent, IDictionary <int, Dictionary <double, double> > dct2DDataProduct, IDictionary <int, float> dct2DDataScanTimes) { // Attempt to parse out the product m/z var parentMZFound = ExtractQ1MZ(chromatogramID, out var parentMZ); var productMZFound = ExtractQ3MZ(chromatogramID, out var productMZ); var spectrumCount = scanTimes.Length; for (var index = 0; index <= spectrumCount - 1; index++) { // Find the ScanNumber in the TIC nearest to scanTimes[index] var indexMatch = FindNearestInList(ticScanTimes, scanTimes[index]); var scanNumber = ticScanNumbers[indexMatch]; // Bump up runtimeMinutes if necessary if (scanTimes[index] > runtimeMinutes) { runtimeMinutes = scanTimes[index]; } var scanStatsEntry = new ScanStatsEntry { ScanNumber = scanNumber, ScanType = 1, ScanTypeName = "SRM", ScanFilterText = StripExtraFromChromatogramID(chromatogramID), ElutionTime = scanTimes[index].ToString("0.0###"), TotalIonIntensity = intensities[index].ToString("0.0"), BasePeakIntensity = intensities[index].ToString("0.0") }; if (parentMZFound) { scanStatsEntry.BasePeakMZ = parentMZ.ToString("0.0###"); } else if (productMZFound) { scanStatsEntry.BasePeakMZ = productMZ.ToString("0.0###"); } else { scanStatsEntry.BasePeakMZ = "0"; } // Base peak signal to noise ratio scanStatsEntry.BasePeakSignalToNoiseRatio = "0"; scanStatsEntry.IonCount = 1; scanStatsEntry.IonCountRaw = 1; mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry); if (mSaveLCMS2DPlots && intensities[index] > 0) { // Store the m/z and intensity values in dct2DDataParent and dct2DDataProduct if (parentMZFound) { Store2DPlotDataPoint(dct2DDataParent, scanNumber, parentMZ, intensities[index]); } if (productMZFound) { Store2DPlotDataPoint(dct2DDataProduct, scanNumber, productMZ, intensities[index]); } if (!dct2DDataScanTimes.ContainsKey(scanNumber)) { dct2DDataScanTimes[scanNumber] = scanTimes[index]; } } } }
/// <summary> /// Examines the subdirectories in the specified zip file /// Determines the oldest and newest modified analysis.baf files (or apexAcquisition.method file if analysis.baf files are not found) /// Presumes this is the AcqStartTime and AcqEndTime /// </summary> /// <param name="zipFile"></param> /// <param name="datasetFileInfo"></param> /// <returns>True if at least one valid file is found; otherwise false</returns> /// <remarks></remarks> private void DetermineAcqStartEndTime(FileInfo zipFile, DatasetFileInfo datasetFileInfo) { var success = false; try { // Bump up the file size datasetFileInfo.FileSizeBytes += zipFile.Length; var fileNamesToFind = new List <string> { "analysis.baf", "apexAcquisition.method", "submethods.xml" }; var zipFileReader = new Ionic.Zip.ZipFile(zipFile.FullName); foreach (var fileNameToFind in fileNamesToFind) { using (var zipEntry = zipFileReader.GetEnumerator()) { while (zipEntry.MoveNext()) { if (zipEntry.Current == null) { continue; } if (zipEntry.Current.IsDirectory) { continue; } // Split the filename on the forward slash character var nameParts = zipEntry.Current.FileName.Split('/'); if (nameParts.Length <= 0) { continue; } if (!string.Equals(nameParts[nameParts.Length - 1], fileNameToFind, StringComparison.OrdinalIgnoreCase)) { continue; } if (zipEntry.Current.LastModified < datasetFileInfo.AcqTimeStart) { datasetFileInfo.AcqTimeStart = zipEntry.Current.LastModified; } if (zipEntry.Current.LastModified > datasetFileInfo.AcqTimeEnd) { datasetFileInfo.AcqTimeEnd = zipEntry.Current.LastModified; } // Bump up the scan count datasetFileInfo.ScanCount += 1; // Add a Scan Stats entry var scanStatsEntry = new ScanStatsEntry { ScanNumber = datasetFileInfo.ScanCount, ScanType = 1, ScanTypeName = "MALDI-HMS", ScanFilterText = string.Empty, ElutionTime = "0", TotalIonIntensity = "0", BasePeakIntensity = "0", BasePeakMZ = "0", BasePeakSignalToNoiseRatio = "0", IonCount = 0, IonCountRaw = 0 }; // Base peak signal to noise ratio mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry); success = true; } } if (success) { break; } } } catch (Exception ex) { OnErrorEvent("Error in DetermineAcqStartEndTime: " + ex.Message); } }
protected void SaveScanStatEntry( StreamWriter writer, clsScanList.eScanTypeConstants eScanType, clsScanInfo currentScan, int datasetID) { const char TAB_DELIMITER = '\t'; var scanStatsEntry = new ScanStatsEntry() { ScanNumber = currentScan.ScanNumber }; if (eScanType == clsScanList.eScanTypeConstants.SurveyScan) { scanStatsEntry.ScanType = 1; scanStatsEntry.ScanTypeName = string.Copy(currentScan.ScanTypeName); } else { if (currentScan.FragScanInfo.MSLevel <= 1) { // This is a fragmentation scan, so it must have a scan type of at least 2 scanStatsEntry.ScanType = 2; } else { // .MSLevel is 2 or higher, record the actual MSLevel value scanStatsEntry.ScanType = currentScan.FragScanInfo.MSLevel; } scanStatsEntry.ScanTypeName = string.Copy(currentScan.ScanTypeName); } scanStatsEntry.ScanFilterText = currentScan.ScanHeaderText; scanStatsEntry.ElutionTime = currentScan.ScanTime.ToString("0.0000"); scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(currentScan.TotalIonIntensity, 5); scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(currentScan.BasePeakIonIntensity, 5); scanStatsEntry.BasePeakMZ = StringUtilities.DblToString(currentScan.BasePeakIonMZ, 4); // Base peak signal to noise ratio scanStatsEntry.BasePeakSignalToNoiseRatio = StringUtilities.ValueToString(MASICPeakFinder.clsMASICPeakFinder.ComputeSignalToNoise(currentScan.BasePeakIonIntensity, currentScan.BaselineNoiseStats.NoiseLevel), 4); scanStatsEntry.IonCount = currentScan.IonCount; scanStatsEntry.IonCountRaw = currentScan.IonCountRaw; mScanTracking.ScanStats.Add(scanStatsEntry); var dataColumns = new List <string>() { datasetID.ToString(), // Dataset ID scanStatsEntry.ScanNumber.ToString(), // Scan number scanStatsEntry.ElutionTime, // Scan time (minutes) scanStatsEntry.ScanType.ToString(), // Scan type (1 for MS, 2 for MS2, etc.) scanStatsEntry.TotalIonIntensity, // Total ion intensity scanStatsEntry.BasePeakIntensity, // Base peak ion intensity scanStatsEntry.BasePeakMZ, // Base peak ion m/z scanStatsEntry.BasePeakSignalToNoiseRatio, // Base peak signal to noise ratio scanStatsEntry.IonCount.ToString(), // Number of peaks (aka ions) in the spectrum scanStatsEntry.IonCountRaw.ToString(), // Number of peaks (aka ions) in the spectrum prior to any filtering scanStatsEntry.ScanTypeName // Scan type name }; writer.WriteLine(string.Join(TAB_DELIMITER.ToString(), dataColumns)); }
private List <udtScansDataType> LoadScansFile(string scansFilePath) { const string FILTERED_SCANS_SUFFIX = "_filtered_scans.csv"; var dctColumnInfo = new Dictionary <string, int> { { "type", -1 }, { "bpi", -1 }, { "bpi_mz", -1 }, { "tic", -1 }, { "num_peaks", -1 }, { "num_deisotoped", -1 } }; var colIndexScanOrFrameNum = -1; var colIndexScanOrFrameTime = -1; var scanData = new List <udtScansDataType>(); var rowNumber = 0; var colIndexScanInfo = -1; if (!File.Exists(scansFilePath) && scansFilePath.ToLower().EndsWith(FILTERED_SCANS_SUFFIX.ToLower())) { scansFilePath = scansFilePath.Substring(0, scansFilePath.Length - FILTERED_SCANS_SUFFIX.Length) + "_scans.csv"; } if (!File.Exists(scansFilePath)) { OnWarningEvent("Warning: _scans.csv file is missing; will plot vs. scan number instead of vs. elution time"); return(scanData); } Console.WriteLine(" Reading the _scans.csv file"); using (var reader = new StreamReader(new FileStream(scansFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) { while (!reader.EndOfStream) { rowNumber += 1; var dataLine = reader.ReadLine(); if (string.IsNullOrWhiteSpace(dataLine)) { continue; } var dataColumns = dataLine.Split(',').ToList(); if (rowNumber == 1) { // Parse the header row ParseColumnHeaders(dctColumnInfo, dataColumns, "_scans.csv"); colIndexScanOrFrameNum = GetScanOrFrameColIndex(dataColumns, "_scans.csv"); colIndexScanOrFrameTime = dataColumns.IndexOf("frame_time"); if (colIndexScanOrFrameTime < 0) { colIndexScanOrFrameTime = dataColumns.IndexOf("scan_time"); } // The info column will have data of the form "FTMS + p NSI Full ms [400.00-2000.00]" for Thermo datasets // For .mzXML files, this fill will simply have an integer (and thus isn't useful) // It may not be present in older _scans.csv files and is thus optional colIndexScanInfo = dataColumns.IndexOf("info"); continue; } try { var udtScanData = new udtScansDataType(); int.TryParse(dataColumns[colIndexScanOrFrameNum], out udtScanData.Scan); float.TryParse(dataColumns[colIndexScanOrFrameTime], out udtScanData.ElutionTime); int.TryParse(dataColumns[dctColumnInfo["type"]], out udtScanData.MSLevel); double.TryParse(dataColumns[dctColumnInfo["bpi"]], out udtScanData.BasePeakIntensity); double.TryParse(dataColumns[dctColumnInfo["bpi_mz"]], out udtScanData.BasePeakMZ); double.TryParse(dataColumns[dctColumnInfo["tic"]], out udtScanData.TotalIonCurrent); int.TryParse(dataColumns[dctColumnInfo["num_peaks"]], out udtScanData.NumPeaks); int.TryParse(dataColumns[dctColumnInfo["num_deisotoped"]], out udtScanData.NumDeisotoped); if (colIndexScanInfo > 0) { var infoText = dataColumns[colIndexScanInfo]; // Only store infoText in .FilterText if infoText is not simply an integer if (!int.TryParse(infoText, out _)) { udtScanData.FilterText = infoText; } } scanData.Add(udtScanData); if (mSaveTICAndBPI) { mTICAndBPIPlot.AddData(udtScanData.Scan, udtScanData.MSLevel, udtScanData.ElutionTime, udtScanData.BasePeakIntensity, udtScanData.TotalIonCurrent); } string scanTypeName; if (string.IsNullOrWhiteSpace(udtScanData.FilterText)) { udtScanData.FilterText = udtScanData.MSLevel > 1 ? "HMSn" : "HMS"; scanTypeName = udtScanData.FilterText; } else { scanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(udtScanData.FilterText); } var scanStatsEntry = new ScanStatsEntry { ScanNumber = udtScanData.Scan, ScanType = udtScanData.MSLevel, ScanTypeName = scanTypeName, ScanFilterText = XRawFileIO.MakeGenericThermoScanFilter(udtScanData.FilterText), ElutionTime = udtScanData.ElutionTime.ToString("0.0###"), TotalIonIntensity = StringUtilities.ValueToString(udtScanData.TotalIonCurrent, 5), BasePeakIntensity = StringUtilities.ValueToString(udtScanData.BasePeakIntensity, 5), BasePeakMZ = udtScanData.BasePeakMZ.ToString("0.0###"), BasePeakSignalToNoiseRatio = "0", IonCount = udtScanData.NumDeisotoped, IonCountRaw = udtScanData.NumPeaks, ExtendedScanInfo = { CollisionMode = string.Empty, ScanFilterText = udtScanData.FilterText } }; mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry); } catch (Exception ex) { OnWarningEvent("Warning: Ignoring scan " + dataColumns[dctColumnInfo["scan_num"]] + " since data conversion error: " + ex.Message); } } } return(scanData); }