private bool ExtractScanInfoWork(
XRawFileIO xcaliburAccessor,
clsScanList scanList,
clsSpectraCache spectraCache,
clsDataOutput dataOutputHandler,
clsSICOptions sicOptions,
ThermoRawFileReader.clsScanInfo thermoScanInfo)
{
if (thermoScanInfo.ParentIonMZ > 0 && Math.Abs(mOptions.ParentIonDecoyMassDa) > 0)
{
thermoScanInfo.ParentIonMZ += mOptions.ParentIonDecoyMassDa;
}
bool success;
// Determine if this was an MS/MS scan
// If yes, determine the scan number of the survey scan
if (thermoScanInfo.MSLevel <= 1)
{
// Survey Scan
success = ExtractXcaliburSurveyScan(xcaliburAccessor, scanList, spectraCache, dataOutputHandler,
sicOptions, thermoScanInfo);
}
else
{
// Fragmentation Scan
success = ExtractXcaliburFragmentationScan(xcaliburAccessor, scanList, spectraCache, dataOutputHandler,
sicOptions, mOptions.BinningOptions, thermoScanInfo);
}
return(success);
}
private void SaveRawDataToDiskWork(
StreamWriter dataWriter,
StreamWriter scanInfoWriter,
clsScanInfo currentScan,
clsSpectraCache spectraCache,
string inputFileName,
bool fragmentationScan,
ref int spectrumExportCount)
{
switch (mOptions.RawDataExportOptions.FileFormat)
{
case clsRawDataExportOptions.eExportRawDataFileFormatConstants.PEKFile:
SavePEKFileToDiskWork(dataWriter, currentScan, spectraCache, inputFileName, fragmentationScan, ref spectrumExportCount);
break;
case clsRawDataExportOptions.eExportRawDataFileFormatConstants.CSVFile:
SaveCSVFilesToDiskWork(dataWriter, scanInfoWriter, currentScan, spectraCache, fragmentationScan, ref spectrumExportCount);
break;
default:
// Unknown format
// This code should never be reached
break;
}
}
private void SaveDecon2LSChromatogram(
ICollection <clsScanInfo> scanList,
clsSpectraCache spectraCache,
string outputFilePath)
{
var scansWritten = 0;
var lastStatus = DateTime.UtcNow;
using (var writer = new StreamWriter(outputFilePath))
{
// Write the file headers
WriteDecon2LSScanFileHeaders(writer);
// Step through the scans and write each one
foreach (var scanItem in scanList)
{
WriteDecon2LSScanFileEntry(writer, scanItem, spectraCache);
if (scansWritten % 250 == 0)
{
UpdateCacheStats(spectraCache);
if (DateTime.UtcNow.Subtract(lastStatus).TotalSeconds >= 30)
{
lastStatus = DateTime.UtcNow;
ReportMessage(string.Format(" {0} / {1} scans processed", scansWritten, scanList.Count));
}
}
scansWritten += 1;
}
}
}
private void WriteDecon2LSScanFileEntry(
StreamWriter writer,
clsScanInfo currentScan,
clsSpectraCache spectraCache)
{
int numPeaks;
if (spectraCache == null)
{
numPeaks = 0;
}
else
{
if (!spectraCache.GetSpectrum(currentScan.ScanNumber, out var spectrum, true))
{
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.ErrorUncachingSpectrum);
return;
}
numPeaks = spectrum.IonCount;
}
var scanNumber = currentScan.ScanNumber;
var msLevel = currentScan.FragScanInfo.MSLevel;
if (msLevel < 1)
{
msLevel = 1;
}
var numIsotopicSignatures = 0;
WriteDecon2LSScanFileEntry(writer, currentScan, scanNumber, msLevel, numPeaks, numIsotopicSignatures);
}
public bool SaveBPIs(
clsScanList scanList,
clsSpectraCache spectraCache,
string inputFilePathFull,
string outputDirectoryPath)
{
// This function creates an ICR-2LS compatible .TIC file (using only the MS1 scans), plus
// two Decon2LS compatible .CSV files (one for the MS1 scans and one for the MS2, MS3, etc. scans)
// Note: Note that SaveExtendedScanStatsFiles() creates a tab-delimited text file with the BPI and TIC information for each scan
var currentFilePath = "_MS_scans.csv";
try
{
var bpiStepCount = 3;
UpdateProgress(0, "Saving chromatograms to disk");
var inputFileName = Path.GetFileName(inputFilePathFull);
// Disabled in April 2015 since not used
// ' First, write a true TIC file (in ICR-2LS format)
// outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, eOutputFileTypeConstants.ICRToolsTICChromatogramByScan)
// LogMessage("Saving ICR Tools TIC to " + Path.GetFileName(outputFilePath))
// SaveICRToolsChromatogramByScan(scanList.SurveyScans, scanList.SurveyScans.Count, outputFilePath, False, True, inputFilePathFull)
var stepsCompleted = 1;
UpdateProgress((short)(stepsCompleted / (double)bpiStepCount * 100));
// Second, write an MS-based _scans.csv file (readable with Decon2LS)
var msScansFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsMSChromatogramFile);
currentFilePath = string.Copy(msScansFilePath);
ReportMessage("Saving Decon2LS MS Chromatogram File to " + Path.GetFileName(msScansFilePath));
SaveDecon2LSChromatogram(scanList.SurveyScans, spectraCache, msScansFilePath);
stepsCompleted += 1;
UpdateProgress((short)(stepsCompleted / (double)bpiStepCount * 100));
// Third, write an MSMS-based _scans.csv file (readable with Decon2LS)
var msmsScansFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsMSMSChromatogramFile);
currentFilePath = string.Copy(msmsScansFilePath);
ReportMessage("Saving Decon2LS MSMS Chromatogram File to " + Path.GetFileName(msmsScansFilePath));
SaveDecon2LSChromatogram(scanList.FragScans, spectraCache, msmsScansFilePath);
UpdateProgress(100);
return(true);
}
catch (Exception ex)
{
ReportError("Error writing the BPI to: " + currentFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError);
return(false);
}
}
public bool XMLOutputFileFinalize(
clsDataOutput dataOutputHandler,
clsScanList scanList,
clsSpectraCache spectraCache,
clsProcessingStats processingStats,
float processingTimeSec)
{
var writer = dataOutputHandler.OutputFileHandles.XMLFileForSICs;
if (writer == null)
{
return(false);
}
try
{
writer.WriteStartElement("ProcessingStats");
writer.WriteElementString("CacheEventCount", spectraCache.CacheEventCount.ToString());
writer.WriteElementString("UnCacheEventCount", spectraCache.UnCacheEventCount.ToString());
writer.WriteElementString("SpectraPoolHitEventCount", spectraCache.SpectraPoolHitEventCount.ToString());
writer.WriteElementString("PeakMemoryUsageMB", StringUtilities.DblToString(processingStats.PeakMemoryUsageMB, 1));
var effectiveSeconds = processingTimeSec - processingStats.TotalProcessingTimeAtStart;
writer.WriteElementString("TotalProcessingTimeSeconds", StringUtilities.DblToString(effectiveSeconds, 1));
writer.WriteEndElement();
if (scanList.ProcessingIncomplete)
{
writer.WriteElementString("ProcessingComplete", "False");
}
else
{
writer.WriteElementString("ProcessingComplete", "True");
}
writer.WriteEndElement(); // Close out the <SICData> element
writer.WriteEndDocument();
writer.Close();
}
catch (Exception ex)
{
ReportError("Error finalizing the XML output file", ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError);
return(false);
}
return(true);
}
private bool ExtractScanInfoCheckRange(
XRawFileIO xcaliburAccessor,
ThermoRawFileReader.clsScanInfo thermoScanInfo,
clsScanList scanList,
clsSpectraCache spectraCache,
clsDataOutput dataOutputHandler,
double percentComplete)
{
bool success;
if (mScanTracking.CheckScanInRange(thermoScanInfo.ScanNumber, thermoScanInfo.RetentionTime, mOptions.SICOptions))
{
success = ExtractScanInfoWork(xcaliburAccessor, scanList, spectraCache, dataOutputHandler,
mOptions.SICOptions, thermoScanInfo);
}
else
{
mScansOutOfRange += 1;
success = true;
}
UpdateProgress((short)Math.Round(percentComplete, 0));
UpdateCacheStats(spectraCache);
if (mOptions.AbortProcessing)
{
scanList.ProcessingIncomplete = true;
return(false);
}
if (DateTime.UtcNow.Subtract(mLastLogTime).TotalSeconds >= 10 ||
thermoScanInfo.ScanNumber % 500 == 0 && (
thermoScanInfo.ScanNumber >= mOptions.SICOptions.ScanRangeStart &&
thermoScanInfo.ScanNumber <= mOptions.SICOptions.ScanRangeEnd))
{
ReportMessage("Reading scan: " + thermoScanInfo.ScanNumber.ToString());
Console.Write(".");
mLastLogTime = DateTime.UtcNow;
}
return(success);
}
public bool ExtractScanInfoFromMGFandCDF(
string filePath,
clsScanList scanList,
clsSpectraCache spectraCache,
DataOutput.clsDataOutput dataOutputHandler,
bool keepRawSpectra,
bool keepMSMSSpectra)
{
// Returns True if Success, False if failure
// Note: This function assumes filePath exists
//
// This function can be used to read a pair of MGF and NetCDF files that contain MS/MS and MS-only parent ion scans, respectively
// Typically, this will apply to LC-MS/MS analyses acquired using an Agilent mass spectrometer running DataAnalysis software
// filePath can contain the path to the MGF or to the CDF file; the extension will be removed in order to determine the base file name,
// then the two files will be looked for separately
var scanTime = 0.0;
var cdfReader = new NetCDFReader.clsMSNetCdf();
var mgfReader = new MSDataFileReader.clsMGFFileReader();
try
{
Console.Write("Reading CDF/MGF data files ");
ReportMessage("Reading CDF/MGF data files");
UpdateProgress(0, "Opening data file: " + Environment.NewLine + Path.GetFileName(filePath));
// Obtain the full path to the file
var mgfFileInfo = new FileInfo(filePath);
var mgfInputFilePathFull = mgfFileInfo.FullName;
// Make sure the extension for mgfInputFilePathFull is .MGF
mgfInputFilePathFull = Path.ChangeExtension(mgfInputFilePathFull, AGILENT_MSMS_FILE_EXTENSION);
var cdfInputFilePathFull = Path.ChangeExtension(mgfInputFilePathFull, AGILENT_MS_FILE_EXTENSION);
var datasetID = mOptions.SICOptions.DatasetID;
var sicOptions = mOptions.SICOptions;
var success = UpdateDatasetFileStats(mgfFileInfo, datasetID);
mDatasetFileInfo.ScanCount = 0;
// Open a handle to each data file
if (!cdfReader.OpenMSCdfFile(cdfInputFilePathFull))
{
ReportError("Error opening input data file: " + cdfInputFilePathFull);
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError);
return(false);
}
if (!mgfReader.OpenFile(mgfInputFilePathFull))
{
ReportError("Error opening input data file: " + mgfInputFilePathFull);
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError);
return(false);
}
var msScanCount = cdfReader.GetScanCount();
mDatasetFileInfo.ScanCount = msScanCount;
if (msScanCount <= 0)
{
// No scans found
ReportError("No scans found in the input file: " + cdfInputFilePathFull);
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError);
return(false);
}
// Reserve memory for all of the Survey Scan data
scanList.Initialize();
var scanCount = mOptions.SICOptions.ScanRangeCount;
if (scanCount <= 0)
{
scanCount = msScanCount * 3;
}
scanList.ReserveListCapacity(scanCount, Math.Min(msScanCount, scanCount / 3)); // Assume there are 2 frag scans for every survey scan
mScanTracking.ReserveListCapacity(scanCount);
spectraCache.SpectrumCount = Math.Max(spectraCache.SpectrumCount, scanCount);
UpdateProgress("Reading CDF/MGF data (" + msScanCount.ToString() + " scans)" + Environment.NewLine + Path.GetFileName(filePath));
ReportMessage("Reading CDF/MGF data; Total MS scan count: " + msScanCount.ToString());
// Read all of the Survey scans from the CDF file
// CDF files created by the Agilent XCT list the first scan number as 0; use scanNumberCorrection to correct for this
var scanNumberCorrection = 0;
for (var msScanIndex = 0; msScanIndex < msScanCount; msScanIndex++)
{
success = cdfReader.GetScanInfo(msScanIndex, out var scanNumber, out var scanTotalIntensity, out scanTime, out _, out _);
if (msScanIndex == 0 && scanNumber == 0)
{
scanNumberCorrection = 1;
}
if (!success)
{
// Error reading CDF file
ReportError("Error obtaining data from CDF file: " + cdfInputFilePathFull);
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileDataReadError);
return(false);
}
if (scanNumberCorrection > 0)
{
scanNumber += scanNumberCorrection;
}
if (mScanTracking.CheckScanInRange(scanNumber, scanTime, sicOptions))
{
var newSurveyScan = new clsScanInfo
{
ScanNumber = scanNumber,
TotalIonIntensity = (float)scanTotalIntensity, // Copy the Total Scan Intensity to .TotalIonIntensity
ScanHeaderText = string.Empty,
ScanTypeName = "MS",
};
if (mOptions.CDFTimeInSeconds)
{
newSurveyScan.ScanTime = (float)(scanTime / 60);
}
else
{
newSurveyScan.ScanTime = (float)scanTime;
}
// Survey scans typically lead to multiple parent ions; we do not record them here
newSurveyScan.FragScanInfo.ParentIonInfoIndex = -1;
scanList.SurveyScans.Add(newSurveyScan);
success = cdfReader.GetMassSpectrum(msScanIndex, out var mzData,
out var intensityData,
out var intIonCount, out _);
if (success && intIonCount > 0)
{
var msSpectrum = new clsMSSpectrum(scanNumber, mzData, intensityData, intIonCount);
double msDataResolution;
newSurveyScan.IonCount = msSpectrum.IonCount;
newSurveyScan.IonCountRaw = newSurveyScan.IonCount;
// Find the base peak ion mass and intensity
newSurveyScan.BasePeakIonMZ = FindBasePeakIon(msSpectrum.IonsMZ,
msSpectrum.IonsIntensity,
out var basePeakIonIntensity,
out var mzMin, out var mzMax);
newSurveyScan.BasePeakIonIntensity = basePeakIonIntensity;
// Determine the minimum positive intensity in this scan
newSurveyScan.MinimumPositiveIntensity = mPeakFinder.FindMinimumPositiveValue(msSpectrum.IonsIntensity, 0);
if (sicOptions.SICToleranceIsPPM)
{
// Define MSDataResolution based on the tolerance value that will be used at the lowest m/z in this spectrum, divided by COMPRESS_TOLERANCE_DIVISOR
// However, if the lowest m/z value is < 100, then use 100 m/z
if (mzMin < 100)
{
msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, 100) /
sicOptions.CompressToleranceDivisorForPPM;
}
else
{
msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, mzMin) /
sicOptions.CompressToleranceDivisorForPPM;
}
}
else
{
msDataResolution = sicOptions.SICTolerance / sicOptions.CompressToleranceDivisorForDa;
}
mScanTracking.ProcessAndStoreSpectrum(
newSurveyScan, this,
spectraCache, msSpectrum,
sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions,
clsMASIC.DISCARD_LOW_INTENSITY_MS_DATA_ON_LOAD,
sicOptions.CompressMSSpectraData,
msDataResolution,
keepRawSpectra);
}
else
{
newSurveyScan.IonCount = 0;
newSurveyScan.IonCountRaw = 0;
}
// Note: Since we're reading all of the Survey Scan data, we cannot update .MasterScanOrder() at this time
}
// Note: We need to take msScanCount * 2 since we have to read two different files
if (msScanCount > 1)
{
UpdateProgress((short)(msScanIndex / (double)(msScanCount * 2 - 1) * 100));
}
else
{
UpdateProgress(0);
}
UpdateCacheStats(spectraCache);
if (mOptions.AbortProcessing)
{
scanList.ProcessingIncomplete = true;
break;
}
if (msScanIndex % 100 == 0)
{
ReportMessage("Reading MS scan index: " + msScanIndex.ToString());
Console.Write(".");
}
}
// Record the current memory usage (before we close the .CDF file)
OnUpdateMemoryUsage();
cdfReader.CloseMSCdfFile();
// We loaded all of the survey scan data above
// We can now initialize .MasterScanOrder()
var lastSurveyScanIndex = 0;
scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, lastSurveyScanIndex);
var surveyScansRecorded = new SortedSet <int>()
{
lastSurveyScanIndex
};
// Reset scanNumberCorrection; we might also apply it to MS/MS data
scanNumberCorrection = 0;
// Now read the MS/MS data from the MGF file
do
{
var fragScanFound = mgfReader.ReadNextSpectrum(out var spectrumInfo);
if (!fragScanFound)
{
break;
}
mDatasetFileInfo.ScanCount += 1;
while (spectrumInfo.ScanNumber < scanList.SurveyScans[lastSurveyScanIndex].ScanNumber)
{
// The scan number for the current MS/MS spectrum is less than the last survey scan index scan number
// This can happen, due to oddities with combining scans when creating the .MGF file
// Need to decrement lastSurveyScanIndex until we find the appropriate survey scan
lastSurveyScanIndex -= 1;
if (lastSurveyScanIndex == 0)
{
break;
}
}
if (scanNumberCorrection == 0)
{
// See if udtSpectrumHeaderInfo.ScanNumberStart is equivalent to one of the survey scan numbers, yielding conflicting scan numbers
// If it is, then there is an indexing error in the .MGF file; this error was present in .MGF files generated with
// an older version of Agilent Chemstation. These files typically have lines like ###MSMS: #13-29 instead of ###MSMS: #13/29/
// If this indexing error is found, then we'll set scanNumberCorrection = 1 and apply it to all subsequent MS/MS scans;
// we'll also need to correct prior MS/MS scans
for (var surveyScanIndex = lastSurveyScanIndex; surveyScanIndex < scanList.SurveyScans.Count; surveyScanIndex++)
{
if (scanList.SurveyScans[surveyScanIndex].ScanNumber == spectrumInfo.ScanNumber)
{
// Conflicting scan numbers were found
scanNumberCorrection = 1;
// Need to update prior MS/MS scans
foreach (var fragScan in scanList.FragScans)
{
fragScan.ScanNumber += scanNumberCorrection;
var scanTimeInterpolated = InterpolateRTandFragScanNumber(
scanList.SurveyScans, 0, fragScan.ScanNumber, out var fragScanIterationOut);
fragScan.FragScanInfo.FragScanNumber = fragScanIterationOut;
fragScan.ScanTime = scanTimeInterpolated;
}
break;
}
if (scanList.SurveyScans[surveyScanIndex].ScanNumber > spectrumInfo.ScanNumber)
{
break;
}
}
}
if (scanNumberCorrection > 0)
{
spectrumInfo.ScanNumber += scanNumberCorrection;
spectrumInfo.ScanNumberEnd += scanNumberCorrection;
}
scanTime = InterpolateRTandFragScanNumber(
scanList.SurveyScans, lastSurveyScanIndex, spectrumInfo.ScanNumber, out var fragScanIteration);
// Make sure this fragmentation scan isn't present yet in scanList.FragScans
// This can occur in Agilent .MGF files if the scan is listed both singly and grouped with other MS/MS scans
var validFragScan = true;
foreach (var fragScan in scanList.FragScans)
{
if (fragScan.ScanNumber == spectrumInfo.ScanNumber)
{
// Duplicate found
validFragScan = false;
break;
}
}
if (!(validFragScan && mScanTracking.CheckScanInRange(spectrumInfo.ScanNumber, scanTime, sicOptions)))
{
continue;
}
// See if lastSurveyScanIndex needs to be updated
// At the same time, populate .MasterScanOrder
while (lastSurveyScanIndex < scanList.SurveyScans.Count - 1 &&
spectrumInfo.ScanNumber > scanList.SurveyScans[lastSurveyScanIndex + 1].ScanNumber)
{
lastSurveyScanIndex += 1;
// Add the given SurveyScan to .MasterScanOrder, though only if it hasn't yet been added
if (!surveyScansRecorded.Contains(lastSurveyScanIndex))
{
surveyScansRecorded.Add(lastSurveyScanIndex);
scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan,
lastSurveyScanIndex);
}
}
scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.FragScan, scanList.FragScans.Count,
spectrumInfo.ScanNumber, (float)scanTime);
var newFragScan = new clsScanInfo
{
ScanNumber = spectrumInfo.ScanNumber,
ScanTime = (float)scanTime,
ScanHeaderText = string.Empty,
ScanTypeName = "MSn",
};
newFragScan.FragScanInfo.FragScanNumber = fragScanIteration;
newFragScan.FragScanInfo.MSLevel = 2;
newFragScan.MRMScanInfo.MRMMassCount = 0;
scanList.FragScans.Add(newFragScan);
var msSpectrum = new clsMSSpectrum(newFragScan.ScanNumber, spectrumInfo.MZList, spectrumInfo.IntensityList, spectrumInfo.DataCount);
if (spectrumInfo.DataCount > 0)
{
newFragScan.IonCount = msSpectrum.IonCount;
newFragScan.IonCountRaw = newFragScan.IonCount;
// Find the base peak ion mass and intensity
newFragScan.BasePeakIonMZ = FindBasePeakIon(msSpectrum.IonsMZ, msSpectrum.IonsIntensity,
out var basePeakIonIntensity,
out _, out _);
newFragScan.BasePeakIonIntensity = basePeakIonIntensity;
// Compute the total scan intensity
newFragScan.TotalIonIntensity = 0;
for (var ionIndex = 0; ionIndex < newFragScan.IonCount; ionIndex++)
{
newFragScan.TotalIonIntensity += msSpectrum.IonsIntensity[ionIndex];
}
// Determine the minimum positive intensity in this scan
newFragScan.MinimumPositiveIntensity = mPeakFinder.FindMinimumPositiveValue(msSpectrum.IonsIntensity, 0);
var msDataResolution = mOptions.BinningOptions.BinSize / sicOptions.CompressToleranceDivisorForDa;
var keepRawSpectrum = keepRawSpectra && keepMSMSSpectra;
mScanTracking.ProcessAndStoreSpectrum(
newFragScan, this,
spectraCache, msSpectrum,
sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions,
clsMASIC.DISCARD_LOW_INTENSITY_MSMS_DATA_ON_LOAD,
sicOptions.CompressMSMSSpectraData,
msDataResolution,
keepRawSpectrum);
}
else
{
newFragScan.IonCount = 0;
newFragScan.IonCountRaw = 0;
newFragScan.TotalIonIntensity = 0;
}
mParentIonProcessor.AddUpdateParentIons(scanList, lastSurveyScanIndex, spectrumInfo.ParentIonMZ,
scanList.FragScans.Count - 1, spectraCache, sicOptions);
// Note: We need to take msScanCount * 2, in addition to adding msScanCount to lastSurveyScanIndex, since we have to read two different files
if (msScanCount > 1)
{
UpdateProgress((short)((lastSurveyScanIndex + msScanCount) / (double)(msScanCount * 2 - 1) * 100));
}
else
{
UpdateProgress(0);
}
UpdateCacheStats(spectraCache);
if (mOptions.AbortProcessing)
{
scanList.ProcessingIncomplete = true;
break;
}
if (scanList.FragScans.Count % 100 == 0)
{
ReportMessage("Reading MSMS scan index: " + scanList.FragScans.Count);
Console.Write(".");
}
}while (true);
// Record the current memory usage (before we close the .MGF file)
OnUpdateMemoryUsage();
mgfReader.CloseFile();
// Check for any other survey scans that need to be added to MasterScanOrder
// See if lastSurveyScanIndex needs to be updated
// At the same time, populate .MasterScanOrder
while (lastSurveyScanIndex < scanList.SurveyScans.Count - 1)
{
lastSurveyScanIndex += 1;
// Note that scanTime is the scan time of the most recent survey scan processed in the above Do loop, so it's not accurate
if (mScanTracking.CheckScanInRange(scanList.SurveyScans[lastSurveyScanIndex].ScanNumber, scanTime, sicOptions))
{
// Add the given SurveyScan to .MasterScanOrder, though only if it hasn't yet been added
if (!surveyScansRecorded.Contains(lastSurveyScanIndex))
{
surveyScansRecorded.Add(lastSurveyScanIndex);
scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, lastSurveyScanIndex);
}
}
}
// Make sure that MasterScanOrder really is sorted by scan number
ValidateMasterScanOrderSorting(scanList);
// Now that all of the data has been read, write out to the scan stats file, in order of scan number
for (var scanIndex = 0; scanIndex < scanList.MasterScanOrderCount; scanIndex++)
{
var eScanType = scanList.MasterScanOrder[scanIndex].ScanType;
clsScanInfo currentScan;
if (eScanType == clsScanList.eScanTypeConstants.SurveyScan)
{
// Survey scan
currentScan = scanList.SurveyScans[scanList.MasterScanOrder[scanIndex].ScanIndexPointer];
}
else
{
// Frag Scan
currentScan = scanList.FragScans[scanList.MasterScanOrder[scanIndex].ScanIndexPointer];
}
SaveScanStatEntry(dataOutputHandler.OutputFileHandles.ScanStats, eScanType, currentScan, sicOptions.DatasetID);
}
Console.WriteLine();
scanList.SetListCapacityToCount();
mScanTracking.SetListCapacityToCount();
return(success);
}
catch (Exception ex)
{
ReportError("Error in ExtractScanInfoFromMGFandCDF", ex, clsMASIC.eMasicErrorCodes.InputFileDataReadError);
return(false);
}
}
private void SavePEKFileToDiskWork(
TextWriter writer,
clsScanInfo currentScan,
clsSpectraCache spectraCache,
string inputFileName,
bool fragmentationScan,
ref int spectrumExportCount)
{
var exportCount = 0;
if (!spectraCache.GetSpectrum(currentScan.ScanNumber, out var spectrum, true))
{
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.ErrorUncachingSpectrum);
return;
}
spectrumExportCount += 1;
writer.WriteLine("Time domain signal level:" + "\t" + currentScan.BasePeakIonIntensity.ToString("0.000")); // Store the base peak ion intensity as the time domain signal level value
writer.WriteLine("MASIC " + mOptions.MASICVersion); // Software version
var dataLine = "MS/MS-based PEK file";
if (mOptions.RawDataExportOptions.IncludeMSMS)
{
dataLine += " (includes both survey scans and fragmentation spectra)";
}
else
{
dataLine += " (includes only survey scans)";
}
writer.WriteLine(dataLine);
int scanNumber;
if (mOptions.RawDataExportOptions.RenumberScans)
{
scanNumber = spectrumExportCount;
}
else
{
scanNumber = currentScan.ScanNumber;
}
dataLine = "Filename: " + inputFileName + "." + scanNumber.ToString("00000");
writer.WriteLine(dataLine);
if (fragmentationScan)
{
writer.WriteLine("ScanType: Fragmentation Scan");
}
else
{
writer.WriteLine("ScanType: Survey Scan");
}
writer.WriteLine("Charge state mass transform results:");
writer.WriteLine("First CS, Number of CS, Abundance, Mass, Standard deviation");
if (spectrum.IonCount > 0)
{
// Populate intensities and pointerArray()
var intensities = new double[spectrum.IonCount];
var pointerArray = new int[spectrum.IonCount];
for (var ionIndex = 0; ionIndex < spectrum.IonCount; ionIndex++)
{
intensities[ionIndex] = spectrum.IonsIntensity[ionIndex];
pointerArray[ionIndex] = ionIndex;
}
// Sort pointerArray() based on the intensities in intensities
Array.Sort(intensities, pointerArray);
int startIndex;
if (mOptions.RawDataExportOptions.MaxIonCountPerScan > 0)
{
// Possibly limit the number of ions to maxIonCount
startIndex = spectrum.IonCount - mOptions.RawDataExportOptions.MaxIonCountPerScan;
if (startIndex < 0)
{
startIndex = 0;
}
}
else
{
startIndex = 0;
}
// Define the minimum data point intensity value
var minimumIntensityCurrentScan = spectrum.IonsIntensity[pointerArray[startIndex]];
// Update the minimum intensity if a higher minimum intensity is defined in .IntensityMinimum
minimumIntensityCurrentScan = Math.Max(minimumIntensityCurrentScan, mOptions.RawDataExportOptions.IntensityMinimum);
// If mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio is > 0, then possibly update minimumIntensityCurrentScan
if (mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio > 0)
{
minimumIntensityCurrentScan = Math.Max(minimumIntensityCurrentScan, currentScan.BaselineNoiseStats.NoiseLevel * mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio);
}
exportCount = 0;
for (var ionIndex = 0; ionIndex < spectrum.IonCount; ionIndex++)
{
if (spectrum.IonsIntensity[ionIndex] >= minimumIntensityCurrentScan)
{
var dataLine1 =
"1" + "\t" +
"1" + "\t" +
spectrum.IonsIntensity[ionIndex] + "\t" +
spectrum.IonsMZ[ionIndex] + "\t" +
"0";
writer.WriteLine(dataLine1);
exportCount += 1;
}
}
}
writer.WriteLine("Number of peaks in spectrum = " + spectrum.IonCount.ToString());
writer.WriteLine("Number of isotopic distributions detected = " + exportCount.ToString());
writer.WriteLine();
}
public bool ExportRawDataToDisk(
clsScanList scanList,
clsSpectraCache spectraCache,
string inputFileName,
string outputDirectoryPath)
{
var outputFilePath = "??";
try
{
StreamWriter dataWriter;
StreamWriter scanInfoWriter;
switch (mOptions.RawDataExportOptions.FileFormat)
{
case clsRawDataExportOptions.eExportRawDataFileFormatConstants.PEKFile:
outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.PEKFile);
dataWriter = new StreamWriter(outputFilePath);
scanInfoWriter = null;
break;
case clsRawDataExportOptions.eExportRawDataFileFormatConstants.CSVFile:
outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsIsosFile);
var outputFilePath2 = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsScansFile);
dataWriter = new StreamWriter(outputFilePath);
scanInfoWriter = new StreamWriter(outputFilePath2);
// Write the file headers
mBPIWriter.WriteDecon2LSIsosFileHeaders(dataWriter);
mBPIWriter.WriteDecon2LSScanFileHeaders(scanInfoWriter);
break;
default:
// Unknown format
ReportError("Unknown raw data file format: " + mOptions.RawDataExportOptions.FileFormat.ToString());
return(false);
}
var spectrumExportCount = 0;
if (!mOptions.RawDataExportOptions.IncludeMSMS && mOptions.RawDataExportOptions.RenumberScans)
{
mOptions.RawDataExportOptions.RenumberScans = true;
}
else
{
mOptions.RawDataExportOptions.RenumberScans = false;
}
UpdateProgress(0, "Exporting raw data");
for (var masterOrderIndex = 0; masterOrderIndex < scanList.MasterScanOrderCount; masterOrderIndex++)
{
var scanPointer = scanList.MasterScanOrder[masterOrderIndex].ScanIndexPointer;
if (scanList.MasterScanOrder[masterOrderIndex].ScanType == clsScanList.eScanTypeConstants.SurveyScan)
{
SaveRawDataToDiskWork(dataWriter, scanInfoWriter, scanList.SurveyScans[scanPointer], spectraCache, inputFileName, false, ref spectrumExportCount);
}
else if (mOptions.RawDataExportOptions.IncludeMSMS ||
scanList.FragScans[scanPointer].MRMScanType != ThermoRawFileReader.MRMScanTypeConstants.NotMRM)
{
// Either we're writing out MS/MS data or this is an MRM scan
SaveRawDataToDiskWork(dataWriter, scanInfoWriter, scanList.FragScans[scanPointer], spectraCache, inputFileName, true, ref spectrumExportCount);
}
if (scanList.MasterScanOrderCount > 1)
{
UpdateProgress((short)(masterOrderIndex / (double)(scanList.MasterScanOrderCount - 1) * 100));
}
else
{
UpdateProgress(0);
}
UpdateCacheStats(spectraCache);
if (mOptions.AbortProcessing)
{
break;
}
}
dataWriter.Close();
scanInfoWriter?.Close();
return(true);
}
catch (Exception ex)
{
ReportError("Error writing the raw spectra data to: " + outputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError);
return(false);
}
}
private void SaveCSVFilesToDiskWork(
StreamWriter dataWriter,
StreamWriter scanInfoWriter,
clsScanInfo currentScan,
clsSpectraCache spectraCache,
bool fragmentationScan,
ref int spectrumExportCount)
{
int scanNumber;
if (!spectraCache.GetSpectrum(currentScan.ScanNumber, out var spectrum, true))
{
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.ErrorUncachingSpectrum);
return;
}
spectrumExportCount += 1;
// First, write an entry to the "_scans.csv" file
if (mOptions.RawDataExportOptions.RenumberScans)
{
scanNumber = spectrumExportCount;
}
else
{
scanNumber = currentScan.ScanNumber;
}
int msLevel;
if (fragmentationScan)
{
msLevel = currentScan.FragScanInfo.MSLevel;
}
else
{
msLevel = 1;
}
var numIsotopicSignatures = 0;
var numPeaks = spectrum.IonCount;
var baselineNoiseLevel = currentScan.BaselineNoiseStats.NoiseLevel;
if (baselineNoiseLevel < 1)
{
baselineNoiseLevel = 1;
}
mBPIWriter.WriteDecon2LSScanFileEntry(scanInfoWriter, currentScan, scanNumber, msLevel, numPeaks, numIsotopicSignatures);
// Now write an entry to the "_isos.csv" file
if (spectrum.IonCount > 0)
{
// Populate intensities and pointerArray()
var intensities = new double[spectrum.IonCount];
var pointerArray = new int[spectrum.IonCount];
for (var ionIndex = 0; ionIndex < spectrum.IonCount; ionIndex++)
{
intensities[ionIndex] = spectrum.IonsIntensity[ionIndex];
pointerArray[ionIndex] = ionIndex;
}
// Sort pointerArray() based on the intensities in intensities
Array.Sort(intensities, pointerArray);
int startIndex;
if (mOptions.RawDataExportOptions.MaxIonCountPerScan > 0)
{
// Possibly limit the number of ions to maxIonCount
startIndex = spectrum.IonCount - mOptions.RawDataExportOptions.MaxIonCountPerScan;
if (startIndex < 0)
{
startIndex = 0;
}
}
else
{
startIndex = 0;
}
// Define the minimum data point intensity value
var minimumIntensityCurrentScan = spectrum.IonsIntensity[pointerArray[startIndex]];
// Update the minimum intensity if a higher minimum intensity is defined in .IntensityMinimum
minimumIntensityCurrentScan = Math.Max(minimumIntensityCurrentScan, mOptions.RawDataExportOptions.IntensityMinimum);
// If mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio is > 0, then possibly update minimumIntensityCurrentScan
if (mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio > 0)
{
minimumIntensityCurrentScan = Math.Max(minimumIntensityCurrentScan, currentScan.BaselineNoiseStats.NoiseLevel * mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio);
}
for (var ionIndex = 0; ionIndex < spectrum.IonCount; ionIndex++)
{
if (spectrum.IonsIntensity[ionIndex] >= minimumIntensityCurrentScan)
{
var charge = 1;
var isoFit = 0;
var mass = clsUtilities.ConvoluteMass(spectrum.IonsMZ[ionIndex], 1, 0);
var peakFWHM = 0;
var signalToNoise = spectrum.IonsIntensity[ionIndex] / baselineNoiseLevel;
var monoisotopicAbu = -10;
var monoPlus2Abu = -10;
mBPIWriter.WriteDecon2LSIsosFileEntry(
dataWriter, scanNumber, charge,
spectrum.IonsIntensity[ionIndex], spectrum.IonsMZ[ionIndex],
isoFit, mass, mass, mass,
peakFWHM, signalToNoise, monoisotopicAbu, monoPlus2Abu);
}
}
}
}
public bool XMLOutputFileInitialize(
string inputFilePathFull,
string outputDirectoryPath,
clsDataOutput dataOutputHandler,
clsScanList scanList,
clsSpectraCache spectraCache,
clsSICOptions sicOptions,
clsBinningOptions binningOptions)
{
var xmlOutputFilePath = string.Empty;
try
{
xmlOutputFilePath = clsDataOutput.ConstructOutputFilePath(inputFilePathFull, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.XMLFile);
dataOutputHandler.OutputFileHandles.XMLFileForSICs = new XmlTextWriter(xmlOutputFilePath, System.Text.Encoding.UTF8);
var writer = dataOutputHandler.OutputFileHandles.XMLFileForSICs;
writer.Formatting = Formatting.Indented;
writer.Indentation = 1;
writer.WriteStartDocument(true);
writer.WriteStartElement("SICData");
writer.WriteStartElement("ProcessingSummary");
writer.WriteElementString("DatasetID", sicOptions.DatasetID.ToString());
writer.WriteElementString("SourceFilePath", inputFilePathFull);
string lastModTimeText;
string fileSizeBytes;
try
{
var inputFileInfo = new FileInfo(inputFilePathFull);
var lastModTime = inputFileInfo.LastWriteTime;
lastModTimeText = lastModTime.ToShortDateString() + " " + lastModTime.ToShortTimeString();
fileSizeBytes = inputFileInfo.Length.ToString();
}
catch (Exception ex)
{
lastModTimeText = string.Empty;
fileSizeBytes = "0";
}
writer.WriteElementString("SourceFileDateTime", lastModTimeText);
writer.WriteElementString("SourceFileSizeBytes", fileSizeBytes);
writer.WriteElementString("MASICProcessingDate", DateTime.Now.ToString(clsDatasetStatsSummarizer.DATE_TIME_FORMAT_STRING));
writer.WriteElementString("MASICVersion", mOptions.MASICVersion);
writer.WriteElementString("MASICPeakFinderDllVersion", mOptions.PeakFinderVersion);
writer.WriteElementString("ScanCountTotal", scanList.MasterScanOrderCount.ToString());
writer.WriteElementString("SurveyScanCount", scanList.SurveyScans.Count.ToString());
writer.WriteElementString("FragScanCount", scanList.FragScans.Count.ToString());
writer.WriteElementString("SkipMSMSProcessing", mOptions.SkipMSMSProcessing.ToString());
writer.WriteElementString("ParentIonDecoyMassDa", mOptions.ParentIonDecoyMassDa.ToString("0.0000"));
writer.WriteEndElement();
writer.WriteStartElement("MemoryOptions");
writer.WriteElementString("CacheAlwaysDisabled", spectraCache.DiskCachingAlwaysDisabled.ToString());
writer.WriteElementString("CacheSpectraToRetainInMemory", spectraCache.CacheSpectraToRetainInMemory.ToString());
writer.WriteEndElement();
writer.WriteStartElement("SICOptions");
// SIC Options
// "SICToleranceDa" is a legacy parameter; If the SIC tolerance is in PPM, then "SICToleranceDa" is the Da tolerance at 1000 m/z
writer.WriteElementString("SICToleranceDa", clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, 1000).ToString("0.0000"));
writer.WriteElementString("SICTolerance", sicOptions.SICTolerance.ToString("0.0000"));
writer.WriteElementString("SICToleranceIsPPM", sicOptions.SICToleranceIsPPM.ToString());
writer.WriteElementString("RefineReportedParentIonMZ", sicOptions.RefineReportedParentIonMZ.ToString());
writer.WriteElementString("ScanRangeStart", sicOptions.ScanRangeStart.ToString());
writer.WriteElementString("ScanRangeEnd", sicOptions.ScanRangeEnd.ToString());
writer.WriteElementString("RTRangeStart", sicOptions.RTRangeStart.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("RTRangeEnd", sicOptions.RTRangeEnd.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("CompressMSSpectraData", sicOptions.CompressMSSpectraData.ToString());
writer.WriteElementString("CompressMSMSSpectraData", sicOptions.CompressMSMSSpectraData.ToString());
writer.WriteElementString("CompressToleranceDivisorForDa", sicOptions.CompressToleranceDivisorForDa.ToString("0.0"));
writer.WriteElementString("CompressToleranceDivisorForPPM", sicOptions.CompressToleranceDivisorForPPM.ToString("0.0"));
writer.WriteElementString("MaxSICPeakWidthMinutesBackward", sicOptions.MaxSICPeakWidthMinutesBackward.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("MaxSICPeakWidthMinutesForward", sicOptions.MaxSICPeakWidthMinutesForward.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("IntensityThresholdFractionMax", StringUtilities.DblToString(sicOptions.SICPeakFinderOptions.IntensityThresholdFractionMax, 5));
writer.WriteElementString("IntensityThresholdAbsoluteMinimum", sicOptions.SICPeakFinderOptions.IntensityThresholdAbsoluteMinimum.ToString(CultureInfo.InvariantCulture));
// Peak Finding Options
var baselineNoiseOptions = sicOptions.SICPeakFinderOptions.SICBaselineNoiseOptions;
writer.WriteElementString("SICNoiseThresholdMode", baselineNoiseOptions.BaselineNoiseMode.ToString());
writer.WriteElementString("SICNoiseThresholdIntensity", baselineNoiseOptions.BaselineNoiseLevelAbsolute.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("SICNoiseFractionLowIntensityDataToAverage", StringUtilities.DblToString(baselineNoiseOptions.TrimmedMeanFractionLowIntensityDataToAverage, 5));
writer.WriteElementString("SICNoiseMinimumSignalToNoiseRatio", baselineNoiseOptions.MinimumSignalToNoiseRatio.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("MaxDistanceScansNoOverlap", sicOptions.SICPeakFinderOptions.MaxDistanceScansNoOverlap.ToString());
writer.WriteElementString("MaxAllowedUpwardSpikeFractionMax", StringUtilities.DblToString(sicOptions.SICPeakFinderOptions.MaxAllowedUpwardSpikeFractionMax, 5));
writer.WriteElementString("InitialPeakWidthScansScaler", sicOptions.SICPeakFinderOptions.InitialPeakWidthScansScaler.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("InitialPeakWidthScansMaximum", sicOptions.SICPeakFinderOptions.InitialPeakWidthScansMaximum.ToString());
writer.WriteElementString("FindPeaksOnSmoothedData", sicOptions.SICPeakFinderOptions.FindPeaksOnSmoothedData.ToString());
writer.WriteElementString("SmoothDataRegardlessOfMinimumPeakWidth", sicOptions.SICPeakFinderOptions.SmoothDataRegardlessOfMinimumPeakWidth.ToString());
writer.WriteElementString("UseButterworthSmooth", sicOptions.SICPeakFinderOptions.UseButterworthSmooth.ToString());
writer.WriteElementString("ButterworthSamplingFrequency", StringUtilities.DblToString(sicOptions.SICPeakFinderOptions.ButterworthSamplingFrequency, 5));
writer.WriteElementString("ButterworthSamplingFrequencyDoubledForSIMData", sicOptions.SICPeakFinderOptions.ButterworthSamplingFrequencyDoubledForSIMData.ToString());
writer.WriteElementString("UseSavitzkyGolaySmooth", sicOptions.SICPeakFinderOptions.UseSavitzkyGolaySmooth.ToString());
writer.WriteElementString("SavitzkyGolayFilterOrder", sicOptions.SICPeakFinderOptions.SavitzkyGolayFilterOrder.ToString());
var noiseThresholdOptions = sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions;
writer.WriteElementString("MassSpectraNoiseThresholdMode", noiseThresholdOptions.BaselineNoiseMode.ToString());
writer.WriteElementString("MassSpectraNoiseThresholdIntensity", noiseThresholdOptions.BaselineNoiseLevelAbsolute.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("MassSpectraNoiseFractionLowIntensityDataToAverage", StringUtilities.DblToString(noiseThresholdOptions.TrimmedMeanFractionLowIntensityDataToAverage, 5));
writer.WriteElementString("MassSpectraNoiseMinimumSignalToNoiseRatio", noiseThresholdOptions.MinimumSignalToNoiseRatio.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("ReplaceSICZeroesWithMinimumPositiveValueFromMSData", sicOptions.ReplaceSICZeroesWithMinimumPositiveValueFromMSData.ToString());
writer.WriteElementString("SaveSmoothedData", sicOptions.SaveSmoothedData.ToString());
// Similarity options
writer.WriteElementString("SimilarIonMZToleranceHalfWidth", sicOptions.SimilarIonMZToleranceHalfWidth.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("SimilarIonToleranceHalfWidthMinutes", sicOptions.SimilarIonToleranceHalfWidthMinutes.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("SpectrumSimilarityMinimum", sicOptions.SpectrumSimilarityMinimum.ToString(CultureInfo.InvariantCulture));
writer.WriteEndElement();
writer.WriteStartElement("BinningOptions");
writer.WriteElementString("BinStartX", binningOptions.StartX.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("BinEndX", binningOptions.EndX.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("BinSize", binningOptions.BinSize.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("MaximumBinCount", binningOptions.MaximumBinCount.ToString());
writer.WriteElementString("IntensityPrecisionPercent", binningOptions.IntensityPrecisionPercent.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("Normalize", binningOptions.Normalize.ToString());
writer.WriteElementString("SumAllIntensitiesForBin", binningOptions.SumAllIntensitiesForBin.ToString());
writer.WriteEndElement();
writer.WriteStartElement("CustomSICValues");
writer.WriteElementString("MZList", mOptions.CustomSICList.RawTextMZList);
writer.WriteElementString("MZToleranceDaList", CheckForEmptyToleranceList(mOptions.CustomSICList.RawTextMZToleranceDaList));
writer.WriteElementString("ScanCenterList", mOptions.CustomSICList.RawTextScanOrAcqTimeCenterList);
writer.WriteElementString("ScanToleranceList", CheckForEmptyToleranceList(mOptions.CustomSICList.RawTextScanOrAcqTimeToleranceList));
writer.WriteElementString("ScanTolerance", mOptions.CustomSICList.ScanOrAcqTimeTolerance.ToString(CultureInfo.InvariantCulture));
writer.WriteElementString("ScanType", mOptions.CustomSICList.ScanToleranceType.ToString());
writer.WriteElementString("LimitSearchToCustomMZList", mOptions.CustomSICList.LimitSearchToCustomMZList.ToString());
writer.WriteEndElement();
}
catch (Exception ex)
{
ReportError("Error initializing the XML output file: " + xmlOutputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError);
return(false);
}
return(true);
}
private bool LoadSpectraForThermoRawFile(
XRawFileIO xcaliburAccessor,
clsSpectraCache spectraCache,
clsScanInfo scanInfo,
clsBaselineNoiseOptions noiseThresholdOptions,
bool discardLowIntensityData,
bool compressSpectraData,
double msDataResolution,
bool keepRawSpectrum)
{
var lastKnownLocation = "Start";
try
{
// Load the ions for this scan
lastKnownLocation = "xcaliburAccessor.GetScanData for scan " + scanInfo.ScanNumber;
// Retrieve the m/z and intensity values for the given scan
// We retrieve the profile-mode data, since that's required for determining spectrum noise
scanInfo.IonCountRaw = xcaliburAccessor.GetScanData(scanInfo.ScanNumber, out var mzList, out var intensityList);
if (scanInfo.IonCountRaw > 0)
{
var ionCountVerified = VerifyDataSorted(scanInfo.ScanNumber, scanInfo.IonCountRaw, mzList, intensityList);
if (ionCountVerified != scanInfo.IonCountRaw)
{
scanInfo.IonCountRaw = ionCountVerified;
}
}
scanInfo.IonCount = scanInfo.IonCountRaw;
lastKnownLocation = "Instantiate new clsMSSpectrum";
var msSpectrum = new clsMSSpectrum(scanInfo.ScanNumber, mzList, intensityList, scanInfo.IonCountRaw);
lastKnownLocation = "Manually determine the base peak m/z and base peak intensity";
// ReSharper disable once CommentTypo
// Regarding BPI, comparison of data read via the ThermoRawFileReader vs.
// that read from the .mzML file for dataset QC_Shew_18_02-run1_02Mar19_Arwen_18-11-02
// showed that 25% of the spectra had incorrect BPI values
double totalIonIntensity = 0;
double basePeakIntensity = 0;
double basePeakMz = 0;
for (var ionIndex = 0; ionIndex < scanInfo.IonCountRaw; ionIndex++)
{
totalIonIntensity += intensityList[ionIndex];
if (intensityList[ionIndex] > basePeakIntensity)
{
basePeakIntensity = intensityList[ionIndex];
basePeakMz = mzList[ionIndex];
}
}
if (Math.Abs(scanInfo.BasePeakIonMZ - basePeakMz) > 0.1)
{
mBpiUpdateCount += 1;
if (mBpiUpdateCount < 10)
{
ConsoleMsgUtils.ShowDebug("Updating BPI in scan {0} from {1:F3} m/z to {2:F3} m/z, and BPI Intensity from {3:F0} to {4:F0}",
scanInfo.ScanNumber, scanInfo.BasePeakIonMZ, basePeakMz, scanInfo.BasePeakIonIntensity, basePeakIntensity);
}
scanInfo.BasePeakIonMZ = basePeakMz;
scanInfo.BasePeakIonIntensity = basePeakIntensity;
}
// Determine the minimum positive intensity in this scan
lastKnownLocation = "Call mMASICPeakFinder.FindMinimumPositiveValue";
scanInfo.MinimumPositiveIntensity = mPeakFinder.FindMinimumPositiveValue(msSpectrum.IonsIntensity, 0);
if (msSpectrum.IonCount > 0)
{
if (scanInfo.TotalIonIntensity < float.Epsilon)
{
scanInfo.TotalIonIntensity = totalIonIntensity;
}
}
else
{
scanInfo.TotalIonIntensity = 0;
}
bool discardLowIntensityDataWork;
bool compressSpectraDataWork;
if (scanInfo.MRMScanType == MRMScanTypeConstants.NotMRM)
{
discardLowIntensityDataWork = discardLowIntensityData;
compressSpectraDataWork = compressSpectraData;
}
else
{
discardLowIntensityDataWork = false;
compressSpectraDataWork = false;
}
lastKnownLocation = "Call ProcessAndStoreSpectrum";
mScanTracking.ProcessAndStoreSpectrum(
scanInfo, this,
spectraCache, msSpectrum,
noiseThresholdOptions,
discardLowIntensityDataWork,
compressSpectraDataWork,
msDataResolution,
keepRawSpectrum);
}
catch (Exception ex)
{
ReportError("Error in LoadSpectraForThermoRawFile (LastKnownLocation: " + lastKnownLocation + ")", ex, clsMASIC.eMasicErrorCodes.InputFileDataReadError);
return(false);
}
return(true);
}
private bool ExtractXcaliburFragmentationScan(
XRawFileIO xcaliburAccessor,
clsScanList scanList,
clsSpectraCache spectraCache,
clsDataOutput dataOutputHandler,
clsSICOptions sicOptions,
clsBinningOptions binningOptions,
ThermoRawFileReader.clsScanInfo thermoScanInfo)
{
// Note that MinimumPositiveIntensity will be determined in LoadSpectraForThermoRawFile
var scanInfo = new clsScanInfo(thermoScanInfo.ParentIonMZ)
{
ScanNumber = thermoScanInfo.ScanNumber,
ScanTime = (float)thermoScanInfo.RetentionTime,
ScanHeaderText = XRawFileIO.MakeGenericThermoScanFilter(thermoScanInfo.FilterText),
ScanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(thermoScanInfo.FilterText),
BasePeakIonMZ = thermoScanInfo.BasePeakMZ,
BasePeakIonIntensity = thermoScanInfo.BasePeakIntensity,
TotalIonIntensity = thermoScanInfo.TotalIonCurrent,
MinimumPositiveIntensity = 0,
ZoomScan = thermoScanInfo.ZoomScan,
SIMScan = thermoScanInfo.SIMScan,
MRMScanType = thermoScanInfo.MRMScanType
};
// Typically .EventNumber is 1 for the parent-ion scan; 2 for 1st frag scan, 3 for 2nd frag scan, etc.
// This resets for each new parent-ion scan
scanInfo.FragScanInfo.FragScanNumber = thermoScanInfo.EventNumber - 1;
// The .EventNumber value is sometimes wrong; need to check for this
// For example, if the dataset only has MS2 scans and no parent-ion scan, .EventNumber will be 2 for every MS2 scan
if (scanList.FragScans.Count > 0)
{
var prevFragScan = scanList.FragScans[scanList.FragScans.Count - 1];
if (prevFragScan.ScanNumber == scanInfo.ScanNumber - 1)
{
if (scanInfo.FragScanInfo.FragScanNumber <= prevFragScan.FragScanInfo.FragScanNumber)
{
scanInfo.FragScanInfo.FragScanNumber = prevFragScan.FragScanInfo.FragScanNumber + 1;
}
}
}
scanInfo.FragScanInfo.MSLevel = thermoScanInfo.MSLevel;
if (scanInfo.MRMScanType != MRMScanTypeConstants.NotMRM)
{
// This is an MRM scan
scanList.MRMDataPresent = true;
scanInfo.MRMScanInfo = clsMRMProcessing.DuplicateMRMInfo(thermoScanInfo.MRMInfo, thermoScanInfo.ParentIonMZ);
if (scanList.SurveyScans.Count == 0)
{
// Need to add a "fake" survey scan that we can map this parent ion to
mLastNonZoomSurveyScanIndex = scanList.AddFakeSurveyScan();
}
}
else
{
scanInfo.MRMScanInfo.MRMMassCount = 0;
}
scanInfo.LowMass = thermoScanInfo.LowMass;
scanInfo.HighMass = thermoScanInfo.HighMass;
scanInfo.IsFTMS = thermoScanInfo.IsFTMS;
// Store the ScanEvent values in .ExtendedHeaderInfo
StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.ScanEvents);
// Store the collision mode and possibly the scan filter text
scanInfo.FragScanInfo.CollisionMode = thermoScanInfo.CollisionMode;
StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_COLLISION_MODE, thermoScanInfo.CollisionMode);
if (mOptions.WriteExtendedStatsIncludeScanFilterText)
{
StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_SCAN_FILTER_TEXT, thermoScanInfo.FilterText);
}
if (mOptions.WriteExtendedStatsStatusLog)
{
// Store the StatusLog values in .ExtendedHeaderInfo
StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.StatusLog, mOptions.StatusLogKeyNameFilterList);
}
scanList.FragScans.Add(scanInfo);
var fragScanIndex = scanList.FragScans.Count - 1;
scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.FragScan, fragScanIndex);
// Note: Even if keepRawSpectra = False, we still need to load the raw data so that we can compute the noise level for the spectrum
var msDataResolution = binningOptions.BinSize / sicOptions.CompressToleranceDivisorForDa;
var success = LoadSpectraForThermoRawFile(
xcaliburAccessor,
spectraCache,
scanInfo,
sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions,
clsMASIC.DISCARD_LOW_INTENSITY_MSMS_DATA_ON_LOAD,
sicOptions.CompressMSMSSpectraData,
msDataResolution,
mKeepRawSpectra && mKeepMSMSSpectra);
if (!success)
{
return(false);
}
SaveScanStatEntry(dataOutputHandler.OutputFileHandles.ScanStats, clsScanList.eScanTypeConstants.FragScan, scanInfo, sicOptions.DatasetID);
if (thermoScanInfo.MRMScanType == MRMScanTypeConstants.NotMRM)
{
// This is not an MRM scan
mParentIonProcessor.AddUpdateParentIons(scanList, mLastNonZoomSurveyScanIndex, thermoScanInfo.ParentIonMZ,
fragScanIndex, spectraCache, sicOptions);
}
else
{
// This is an MRM scan
mParentIonProcessor.AddUpdateParentIons(scanList, mLastNonZoomSurveyScanIndex, thermoScanInfo.ParentIonMZ,
scanInfo.MRMScanInfo, spectraCache, sicOptions);
}
if (mLastNonZoomSurveyScanIndex >= 0)
{
var precursorScanNumber = scanList.SurveyScans[mLastNonZoomSurveyScanIndex].ScanNumber;
// Compute the interference of the parent ion in the MS1 spectrum for this frag scan
scanInfo.FragScanInfo.InterferenceScore = ComputeInterference(xcaliburAccessor, thermoScanInfo, precursorScanNumber);
}
return(true);
}
private bool ExtractXcaliburSurveyScan(
XRawFileIO xcaliburAccessor,
clsScanList scanList,
clsSpectraCache spectraCache,
clsDataOutput dataOutputHandler,
clsSICOptions sicOptions,
ThermoRawFileReader.clsScanInfo thermoScanInfo)
{
var scanInfo = new clsScanInfo()
{
ScanNumber = thermoScanInfo.ScanNumber,
ScanTime = (float)thermoScanInfo.RetentionTime,
ScanHeaderText = XRawFileIO.MakeGenericThermoScanFilter(thermoScanInfo.FilterText),
ScanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(thermoScanInfo.FilterText),
BasePeakIonMZ = thermoScanInfo.BasePeakMZ,
BasePeakIonIntensity = thermoScanInfo.BasePeakIntensity,
TotalIonIntensity = thermoScanInfo.TotalIonCurrent,
MinimumPositiveIntensity = 0, // This will be determined in LoadSpectraForThermoRawFile
ZoomScan = thermoScanInfo.ZoomScan,
SIMScan = thermoScanInfo.SIMScan,
MRMScanType = thermoScanInfo.MRMScanType,
LowMass = thermoScanInfo.LowMass,
HighMass = thermoScanInfo.HighMass,
IsFTMS = thermoScanInfo.IsFTMS
};
// Survey scans typically lead to multiple parent ions; we do not record them here
scanInfo.FragScanInfo.ParentIonInfoIndex = -1;
if (scanInfo.MRMScanType != MRMScanTypeConstants.NotMRM)
{
// This is an MRM scan
scanList.MRMDataPresent = true;
}
if (scanInfo.SIMScan)
{
scanList.SIMDataPresent = true;
var simKey = scanInfo.LowMass + "_" + scanInfo.HighMass;
if (mSIMScanMapping.TryGetValue(simKey, out var simIndex))
{
scanInfo.SIMIndex = simIndex;
}
else
{
scanInfo.SIMIndex = mSIMScanMapping.Count;
mSIMScanMapping.Add(simKey, mSIMScanMapping.Count);
}
}
// Store the ScanEvent values in .ExtendedHeaderInfo
StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.ScanEvents);
// Store the collision mode and possibly the scan filter text
scanInfo.FragScanInfo.CollisionMode = thermoScanInfo.CollisionMode;
StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_COLLISION_MODE, thermoScanInfo.CollisionMode);
if (mOptions.WriteExtendedStatsIncludeScanFilterText)
{
StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_SCAN_FILTER_TEXT, thermoScanInfo.FilterText);
}
if (mOptions.WriteExtendedStatsStatusLog)
{
// Store the StatusLog values in .ExtendedHeaderInfo
StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.StatusLog, mOptions.StatusLogKeyNameFilterList);
}
scanList.SurveyScans.Add(scanInfo);
if (!scanInfo.ZoomScan)
{
mLastNonZoomSurveyScanIndex = scanList.SurveyScans.Count - 1;
}
scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, scanList.SurveyScans.Count - 1);
double msDataResolution;
if (sicOptions.SICToleranceIsPPM)
{
// Define MSDataResolution based on the tolerance value that will be used at the lowest m/z in this spectrum, divided by sicOptions.CompressToleranceDivisorForPPM
// However, if the lowest m/z value is < 100, then use 100 m/z
if (thermoScanInfo.LowMass < 100)
{
msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, 100) /
sicOptions.CompressToleranceDivisorForPPM;
}
else
{
msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, thermoScanInfo.LowMass) /
sicOptions.CompressToleranceDivisorForPPM;
}
}
else
{
msDataResolution = sicOptions.SICTolerance / sicOptions.CompressToleranceDivisorForDa;
}
// Note: Even if mKeepRawSpectra = False, we still need to load the raw data so that we can compute the noise level for the spectrum
var success = LoadSpectraForThermoRawFile(
xcaliburAccessor,
spectraCache,
scanInfo,
sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions,
clsMASIC.DISCARD_LOW_INTENSITY_MS_DATA_ON_LOAD,
sicOptions.CompressMSSpectraData,
msDataResolution,
mKeepRawSpectra);
if (!success)
{
return(false);
}
SaveScanStatEntry(dataOutputHandler.OutputFileHandles.ScanStats, clsScanList.eScanTypeConstants.SurveyScan, scanInfo, sicOptions.DatasetID);
return(true);
}
/// <summary>
/// Read scan data and ions from a Thermo .raw file
/// </summary>
/// <param name="filePath"></param>
/// <param name="scanList"></param>
/// <param name="spectraCache"></param>
/// <param name="dataOutputHandler"></param>
/// <param name="keepRawSpectra"></param>
/// <param name="keepMSMSSpectra"></param>
/// <returns>True if Success, False if failure</returns>
/// <remarks>Assumes filePath exists</remarks>
public bool ExtractScanInfoFromXcaliburDataFile(
string filePath,
clsScanList scanList,
clsSpectraCache spectraCache,
clsDataOutput dataOutputHandler,
bool keepRawSpectra,
bool keepMSMSSpectra)
{
// Use XrawFileIO to read the .Raw files (it uses ThermoFisher.CommonCore)
var readerOptions = new ThermoReaderOptions
{
LoadMSMethodInfo = mOptions.WriteMSMethodFile,
LoadMSTuneInfo = mOptions.WriteMSTuneFile
};
var xcaliburAccessor = new XRawFileIO(readerOptions)
{
ScanInfoCacheMaxSize = 0 // Don't cache scanInfo objects
};
RegisterEvents(xcaliburAccessor);
mBpiUpdateCount = 0;
// Assume success for now
var success = true;
try
{
Console.Write("Reading Thermo .raw file ");
ReportMessage("Reading Thermo .raw file");
UpdateProgress(0, "Opening data file:" + Environment.NewLine + Path.GetFileName(filePath));
// Obtain the full path to the file
var rawFileInfo = new FileInfo(filePath);
var inputFileFullPath = rawFileInfo.FullName;
// Open a handle to the data file
if (!xcaliburAccessor.OpenRawFile(inputFileFullPath))
{
ReportError("Error opening input data file: " + inputFileFullPath + " (xcaliburAccessor.OpenRawFile returned False)");
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError);
return(false);
}
var datasetID = mOptions.SICOptions.DatasetID;
success = UpdateDatasetFileStats(rawFileInfo, datasetID, xcaliburAccessor);
var metadataWriter = new clsThermoMetadataWriter();
RegisterEvents(metadataWriter);
if (mOptions.WriteMSMethodFile)
{
metadataWriter.SaveMSMethodFile(xcaliburAccessor, dataOutputHandler);
}
if (mOptions.WriteMSTuneFile)
{
metadataWriter.SaveMSTuneFile(xcaliburAccessor, dataOutputHandler);
}
var scanCount = xcaliburAccessor.GetNumScans();
if (scanCount <= 0)
{
// No scans found
ReportError("No scans found in the input file: " + filePath);
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError);
return(false);
}
var scanStart = xcaliburAccessor.ScanStart;
var scanEnd = xcaliburAccessor.ScanEnd;
InitOptions(scanList, keepRawSpectra, keepMSMSSpectra);
UpdateProgress(string.Format("Reading Xcalibur data ({0:N0} scans){1}", scanCount, Environment.NewLine + Path.GetFileName(filePath)));
ReportMessage(string.Format("Reading Xcalibur data; Total scan count: {0:N0}", scanCount));
var scanCountToRead = scanEnd - scanStart + 1;
var scansEst = mOptions.SICOptions.ScanRangeCount;
if (scansEst <= 0)
{
scansEst = scanCountToRead;
}
scanList.ReserveListCapacity(scansEst);
mScanTracking.ReserveListCapacity(scansEst);
spectraCache.SpectrumCount = Math.Max(spectraCache.SpectrumCount, scansEst);
for (var scanNumber = scanStart; scanNumber <= scanEnd; scanNumber++)
{
if (!mScanTracking.CheckScanInRange(scanNumber, mOptions.SICOptions))
{
mScansOutOfRange += 1;
continue;
}
success = xcaliburAccessor.GetScanInfo(scanNumber, out ThermoRawFileReader.clsScanInfo thermoScanInfo);
if (!success)
{
// GetScanInfo returned false
ReportWarning("xcaliburAccessor.GetScanInfo returned false for scan " + scanNumber.ToString() + "; aborting read");
break;
}
var percentComplete = scanList.MasterScanOrderCount / (double)(scanCountToRead) * 100;
var extractSuccess = ExtractScanInfoCheckRange(xcaliburAccessor, thermoScanInfo, scanList, spectraCache, dataOutputHandler, percentComplete);
if (!extractSuccess)
{
break;
}
}
Console.WriteLine();
scanList.SetListCapacityToCount();
mScanTracking.SetListCapacityToCount();
// Shrink the memory usage of the scanList arrays
success = FinalizeScanList(scanList, rawFileInfo);
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
ReportError("Error in ExtractScanInfoFromXcaliburDataFile", ex, clsMASIC.eMasicErrorCodes.InputFileDataReadError);
}
// Close the handle to the data file
xcaliburAccessor.CloseRawFile();
return(success);
}
