public void TestPWiz(
string fileOrDirectoryName,
bool isDirectory,
int expectedSpectraInFile,
int expectedSpectraLoaded,
int expectedTotalDataPoints,
double expectedMedianTIC,
double expectedMedianBPI)
{
try
{
if (!InstrumentDataUtilities.FindInstrumentData(fileOrDirectoryName, isDirectory, out var instrumentDataFileOrDirectory))
{
Assert.Fail("File or directory not found");
return;
}
using (var reader = new MSDataFileReader(instrumentDataFileOrDirectory.FullName))
{
Console.WriteLine("Chromatogram count: " + reader.ChromatogramCount);
Console.WriteLine();
var ticIntensities = new Dictionary <int, float>();
for (var chromatogramIndex = 0; chromatogramIndex < reader.ChromatogramCount; chromatogramIndex++)
{
// Note that even for a small .Wiff file (1.5 MB), obtaining the Chromatogram list will take some time (20 to 60 seconds)
// The chromatogram at index 0 should be the TIC
// The chromatogram at index >=1 will be each SRM
reader.GetChromatogram(chromatogramIndex, out var chromatogramID, out var timeArray, out var intensityArray);
// Determine the chromatogram type
if (chromatogramID == null)
{
chromatogramID = string.Empty;
}
var cvParams = reader.GetChromatogramCVParams(chromatogramIndex);
if (MSDataFileReader.TryGetCVParamDouble(cvParams, pwiz.CLI.cv.CVID.MS_TIC_chromatogram, out _))
{
// This chromatogram is the TIC
Console.WriteLine("TIC has id {0} and {1} data points", chromatogramID, timeArray.Length);
for (var i = 0; i < intensityArray.Length; i++)
{
ticIntensities.Add(i, intensityArray[i]);
}
}
if (MSDataFileReader.TryGetCVParamDouble(cvParams, pwiz.CLI.cv.CVID.MS_selected_reaction_monitoring_chromatogram, out _))
{
// This chromatogram is an SRM scan
Console.WriteLine("SRM scan has id {0} and {1} data points", chromatogramID, timeArray.Length);
}
}
Console.WriteLine("Spectrum count: " + reader.SpectrumCount);
var spectraLoaded = 0;
long totalPointsRead = 0;
double ticSumAllSpectra = 0;
double bpiSumAllSpectra = 0;
var spectrumIndex = 0;
while (spectrumIndex < reader.SpectrumCount)
{
var spectrum = reader.GetSpectrum(spectrumIndex, getBinaryData: true);
spectraLoaded += 1;
Console.WriteLine();
Console.WriteLine("ScanIndex {0}, NativeId {1}, Elution Time {2:F2} minutes, MS Level {3}",
spectrumIndex, spectrum.NativeId, spectrum.RetentionTime, spectrum.Level);
// Use the following to get the MZs and Intensities
var mzList = spectrum.Mzs.ToList();
var intensities = spectrum.Intensities.ToList();
if (mzList.Count > 0)
{
Console.WriteLine(" Data count: " + mzList.Count);
totalPointsRead += mzList.Count;
double tic = 0;
double bpi = 0;
for (var index = 0; index <= mzList.Count - 1; index++)
{
tic += intensities[index];
if (intensities[index] > bpi)
{
bpi = intensities[index];
}
}
ticSumAllSpectra += tic;
bpiSumAllSpectra += bpi;
if (!ticIntensities.TryGetValue(spectrumIndex, out var ticFromChromatogram))
{
ticFromChromatogram = -1;
}
var spectrumInfo = reader.GetSpectrumObject(spectrumIndex);
if (MSDataFileReader.TryGetCVParamDouble(spectrumInfo.cvParams, pwiz.CLI.cv.CVID.MS_total_ion_current,
out var ticFromSpectrumObject))
{
if (ticFromChromatogram < 0)
{
// ticIntensities did not have an entry for spectrumIndex
// This could be the case on a Waters Synapt instrument, where ticIntensities has one TIC per frame,
// while pWiz.GetSpectrum() returns individual mass spectra, of which there could be hundreds of spectra per frame
Console.WriteLine(" TIC from actual data is {0:E2} vs. {1:E2} from the spectrum object",
tic, ticFromSpectrumObject);
}
else
{
// Note: the TIC value from the CvParams has been seen to be drastically off from the manually computed value
Console.WriteLine(
" TIC from actual data is {0:E2} vs. {1:E2} from the chromatogram and {2:E2} from the spectrum object",
tic, ticFromChromatogram, ticFromSpectrumObject);
}
}
if (MSDataFileReader.TryGetCVParamDouble(spectrumInfo.cvParams, pwiz.CLI.cv.CVID.MS_base_peak_intensity,
out var bpiFromSpectrumObject))
{
if (MSDataFileReader.TryGetCVParamDouble(spectrumInfo.cvParams, pwiz.CLI.cv.CVID.MS_base_peak_m_z,
out var bpiMzFromSpectrumObject))
{
// Note: the BPI intensity from the CvParams has been seen to be drastically off from the manually computed value
Console.WriteLine(" BPI from spectrum object is {0:E2} at {1:F3} m/z",
bpiFromSpectrumObject, bpiMzFromSpectrumObject);
}
}
}
if (spectrumIndex < 25)
{
spectrumIndex += 1;
}
else if (spectrumIndex < 1250)
{
spectrumIndex += 50;
}
else
{
spectrumIndex += 500;
}
}
if (spectraLoaded > 0)
{
var medianTIC = ticSumAllSpectra / spectraLoaded;
var medianBPI = bpiSumAllSpectra / spectraLoaded;
Console.WriteLine();
Console.WriteLine("Read {0:N0} data points from {1} spectra in {2}",
totalPointsRead, spectraLoaded, Path.GetFileName(fileOrDirectoryName));
Console.WriteLine("Median TIC: {0:E4}", medianTIC);
Console.WriteLine("Median BPI: {0:E4}", medianBPI);
Assert.AreEqual(expectedSpectraInFile, reader.SpectrumCount, "Total spectrum count mismatch");
Assert.AreEqual(expectedSpectraLoaded, spectraLoaded, "Spectra loaded mismatch");
Assert.AreEqual(expectedTotalDataPoints, totalPointsRead, "Spectra loaded mismatch");
var ticComparisonTolerance = expectedMedianTIC * 0.01;
var bpiComparisonTolerance = expectedMedianBPI * 0.01;
Assert.AreEqual(expectedMedianTIC, medianTIC, ticComparisonTolerance, "Median TIC mismatch");
Assert.AreEqual(expectedMedianBPI, medianBPI, bpiComparisonTolerance, "Median BPI mismatch");
}
}
}
catch (Exception ex)
{
ConsoleMsgUtils.ShowError("Error using ProteoWizard reader", ex);
}
}
public bool StoreMSSpectraInfo(
bool ticStored,
ref double runtimeMinutes,
bool skipExistingScans,
bool skipScansWithNoIons,
int maxScansToTrackInDetail,
int maxScansForTicAndBpi,
out int scanCountSuccess,
out int scanCountError)
{
scanCountSuccess = 0;
scanCountError = 0;
try
{
Console.WriteLine();
OnStatusEvent("Obtaining scan times and MSLevels (this could take several minutes)");
mLastScanLoadingDebugProgressTime = DateTime.UtcNow;
mLastScanLoadingStatusProgressTime = DateTime.UtcNow;
mReportedTotalSpectraToExamine = false;
mCancellationToken = new CancellationTokenSource();
var scanTimes = new double[0];
var msLevels = new byte[0];
mGetScanTimesStartTime = DateTime.UtcNow;
mGetScanTimesMaxWaitTimeSeconds = 90;
mGetScanTimesAutoAborted = false;
var minScanIndexWithoutScanTimes = int.MaxValue;
try
{
mPWiz.GetScanTimesAndMsLevels(mCancellationToken.Token, out scanTimes, out msLevels, MonitorScanTimeLoadingProgress);
}
catch (OperationCanceledException)
{
// mCancellationToken.Cancel was called in MonitorScanTimeLoadingProgress
// Determine the scan index where GetScanTimesAndMsLevels exited the for loop
for (var scanIndex = 0; scanIndex <= scanTimes.Length - 1; scanIndex++)
{
if (msLevels[scanIndex] > 0)
{
continue;
}
minScanIndexWithoutScanTimes = scanIndex;
break;
}
if (!mGetScanTimesAutoAborted && minScanIndexWithoutScanTimes < int.MaxValue)
{
// Manually aborted; shrink the arrays to reflect the amount of data that was actually loaded
Array.Resize(ref scanTimes, minScanIndexWithoutScanTimes);
Array.Resize(ref msLevels, minScanIndexWithoutScanTimes);
}
}
var spectrumCount = scanTimes.Length;
// The scan times returned by .GetScanTimesAndMsLevels() are the acquisition time in seconds from the start of the analysis
// Convert these to minutes
for (var scanIndex = 0; scanIndex <= spectrumCount - 1; scanIndex++)
{
if (scanIndex >= minScanIndexWithoutScanTimes)
{
break;
}
var scanTimeMinutes = scanTimes[scanIndex] / 60.0;
scanTimes[scanIndex] = scanTimeMinutes;
}
Console.WriteLine();
OnStatusEvent("Reading spectra");
var lastDebugProgressTime = DateTime.UtcNow;
var lastStatusProgressTime = DateTime.UtcNow;
var skippedEmptyScans = 0;
var scanNumber = 0;
var scansStored = 0;
var ticAndBpiScansStored = 0;
var scanCountHMS = 0;
var scanCountHMSn = 0;
var scanCountMS = 0;
var scanCountMSn = 0;
for (var scanIndex = 0; scanIndex <= spectrumCount - 1; scanIndex++)
{
try
{
var computeTIC = true;
var computeBPI = true;
// Obtain the raw mass spectrum
var msDataSpectrum = mPWiz.GetSpectrum(scanIndex);
scanNumber = scanIndex + 1;
if (scanIndex >= minScanIndexWithoutScanTimes)
{
// msDataSpectrum.RetentionTime is already in minutes
scanTimes[scanIndex] = msDataSpectrum.RetentionTime ?? 0;
if (msDataSpectrum.Level >= byte.MinValue && msDataSpectrum.Level <= byte.MaxValue)
{
msLevels[scanIndex] = (byte)msDataSpectrum.Level;
}
}
var scanStatsEntry = new ScanStatsEntry
{
ScanNumber = scanNumber,
ScanType = msDataSpectrum.Level
};
if (msLevels[scanIndex] > 1)
{
if (HighResMS2)
{
scanStatsEntry.ScanTypeName = "HMSn";
scanCountHMSn++;
}
else
{
scanStatsEntry.ScanTypeName = "MSn";
scanCountMSn++;
}
}
else
{
if (HighResMS1)
{
scanStatsEntry.ScanTypeName = "HMS";
scanCountHMS++;
}
else
{
scanStatsEntry.ScanTypeName = "MS";
scanCountMS++;
}
}
var driftTimeMsec = msDataSpectrum.DriftTimeMsec ?? 0;
scanStatsEntry.ScanFilterText = driftTimeMsec > 0 ? "IMS" : string.Empty;
scanStatsEntry.ExtendedScanInfo.ScanFilterText = scanStatsEntry.ScanFilterText;
scanStatsEntry.DriftTimeMsec = driftTimeMsec.ToString("0.0###");
scanStatsEntry.ElutionTime = scanTimes[scanIndex].ToString("0.0###");
// Bump up runtimeMinutes if necessary
if (scanTimes[scanIndex] > runtimeMinutes)
{
runtimeMinutes = scanTimes[scanIndex];
}
var spectrum = mPWiz.GetSpectrumObject(scanIndex);
if (MSDataFileReader.TryGetCVParamDouble(spectrum.cvParams, pwiz.CLI.cv.CVID.MS_total_ion_current, out var tic))
{
// For timsTOF data, this is the TIC of the entire frame
scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(tic, 5);
computeTIC = false;
}
if (MSDataFileReader.TryGetCVParamDouble(spectrum.cvParams, pwiz.CLI.cv.CVID.MS_base_peak_intensity, out var bpi))
{
// For timsTOF data, this is the BPI of the entire frame
// Additionally, for timsTOF data, MS_base_peak_m_z is not defined
scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(bpi, 5);
if (MSDataFileReader.TryGetCVParamDouble(spectrum.scanList.scans[0].cvParams, pwiz.CLI.cv.CVID.MS_base_peak_m_z, out var basePeakMzFromCvParams))
{
scanStatsEntry.BasePeakMZ = StringUtilities.ValueToString(basePeakMzFromCvParams, 5);
computeBPI = false;
}
}
if (string.IsNullOrEmpty(scanStatsEntry.ScanFilterText))
{
if (spectrum?.scanList?.scans.Count > 0)
{
// Bruker timsTOF datasets will have CVParam "inverse reduced ion mobility" for IMS spectra; check for this
foreach (var scanItem in spectrum.scanList.scans)
{
if (MSDataFileReader.TryGetCVParam(scanItem.cvParams, pwiz.CLI.cv.CVID.MS_inverse_reduced_ion_mobility, out _))
{
scanStatsEntry.ScanFilterText = "IMS";
break;
}
}
}
}
// Base peak signal to noise ratio
scanStatsEntry.BasePeakSignalToNoiseRatio = "0";
scanStatsEntry.IonCount = msDataSpectrum.Mzs.Length;
scanStatsEntry.IonCountRaw = scanStatsEntry.IonCount;
if ((computeBPI || computeTIC) && scanStatsEntry.IonCount > 0)
{
// Step through the raw data to compute the BPI and TIC
var mzList = msDataSpectrum.Mzs;
var intensities = msDataSpectrum.Intensities;
tic = 0;
bpi = 0;
double basePeakMZ = 0;
for (var index = 0; index <= mzList.Length - 1; index++)
{
tic += intensities[index];
if (intensities[index] > bpi)
{
bpi = intensities[index];
basePeakMZ = mzList[index];
}
}
scanStatsEntry.TotalIonIntensity = StringUtilities.ValueToString(tic, 5);
scanStatsEntry.BasePeakIntensity = StringUtilities.ValueToString(bpi, 5);
scanStatsEntry.BasePeakMZ = StringUtilities.ValueToString(basePeakMZ, 5);
}
var addScan = !skipExistingScans || skipExistingScans && !mDatasetStatsSummarizer.HasScanNumber(scanNumber);
if (addScan)
{
if (skipScansWithNoIons && scanStatsEntry.IonCount == 0)
{
skippedEmptyScans++;
if (skippedEmptyScans < 25 ||
skippedEmptyScans < 100 && skippedEmptyScans % 10 == 0 ||
skippedEmptyScans < 1000 && skippedEmptyScans % 100 == 0 ||
skippedEmptyScans < 10000 && skippedEmptyScans % 1000 == 0 ||
skippedEmptyScans < 100000 && skippedEmptyScans % 10000 == 0 ||
skippedEmptyScans % 100000 == 0)
{
ConsoleMsgUtils.ShowDebug("Skipping scan {0:N0} since no ions; {1:N0} total skipped scans", scanNumber, skippedEmptyScans);
}
}
else
{
if (maxScansToTrackInDetail < 0 || scansStored < maxScansToTrackInDetail)
{
mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry);
scansStored += 1;
}
}
}
if (mSaveTICAndBPI && !ticStored && (maxScansForTicAndBpi < 0 || ticAndBpiScansStored < maxScansForTicAndBpi))
{
mTICAndBPIPlot.AddData(scanStatsEntry.ScanNumber, msLevels[scanIndex], (float)scanTimes[scanIndex], bpi, tic);
ticAndBpiScansStored += 1;
}
if (mSaveLCMS2DPlots && addScan)
{
mLCMS2DPlot.AddScan(scanStatsEntry.ScanNumber, msLevels[scanIndex], (float)scanTimes[scanIndex], msDataSpectrum.Mzs.Length, msDataSpectrum.Mzs, msDataSpectrum.Intensities);
}
if (mCheckCentroidingStatus)
{
mDatasetStatsSummarizer.ClassifySpectrum(msDataSpectrum.Mzs, msLevels[scanIndex], "Scan " + scanStatsEntry.ScanNumber);
}
scanCountSuccess += 1;
}
catch (Exception ex)
{
OnErrorEvent("Error loading header info for scan " + scanNumber, ex);
scanCountError += 1;
}
if (DateTime.UtcNow.Subtract(lastStatusProgressTime).TotalMinutes > 5)
{
OnStatusEvent(string.Format("Reading spectra, loaded {0:N0} / {1:N0} spectra; " +
"{2:N0} HMS spectra; {3:N0} HMSn spectra; " +
"{4:N0} MS spectra; {5:N0} MSn spectra; " +
"max elution time is {6:F2} minutes",
scanNumber, spectrumCount,
scanCountHMS, scanCountHMSn,
scanCountMS, scanCountMSn,
runtimeMinutes));
lastStatusProgressTime = DateTime.UtcNow;
lastDebugProgressTime = DateTime.UtcNow;
continue;
}
if (DateTime.UtcNow.Subtract(lastDebugProgressTime).TotalSeconds < 15)
{
continue;
}
lastDebugProgressTime = DateTime.UtcNow;
var percentComplete = scanNumber / (float)spectrumCount * 100;
var percentCompleteOverall = clsMSFileInfoProcessorBaseClass.ComputeIncrementalProgress(
PROGRESS_SCAN_TIMES_LOADED,
clsMSFileInfoProcessorBaseClass.PROGRESS_SPECTRA_LOADED,
percentComplete);
OnProgressUpdate(string.Format("Spectra processed: {0:N0}", scanNumber), percentCompleteOverall);
}
mDatasetStatsSummarizer.StoreScanTypeTotals(scanCountHMS, scanCountHMSn,
scanCountMS, scanCountMSn,
runtimeMinutes);
var scanCountTotal = scanCountSuccess + scanCountError;
if (scanCountTotal == 0)
{
return(false);
}
// Return True if at least 50% of the spectra were successfully read
return(scanCountSuccess >= scanCountTotal / 2.0);
}
catch (AccessViolationException)
{
// Attempted to read or write protected memory. This is often an indication that other memory is corrupt.
if (!mWarnedAccessViolationException)
{
OnWarningEvent("Error reading instrument data with ProteoWizard: Attempted to read or write protected memory. " +
"The instrument data file is likely corrupt.");
mWarnedAccessViolationException = true;
}
mDatasetStatsSummarizer.CreateEmptyScanStatsFiles = false;
return(false);
}
catch (Exception ex)
{
OnErrorEvent("Error reading instrument data with ProteoWizard: " + ex.Message, ex);
return(false);
}
}