private bool ExtractScanInfoWork( XRawFileIO xcaliburAccessor, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, clsSICOptions sicOptions, ThermoRawFileReader.clsScanInfo thermoScanInfo) { if (thermoScanInfo.ParentIonMZ > 0 && Math.Abs(mOptions.ParentIonDecoyMassDa) > 0) { thermoScanInfo.ParentIonMZ += mOptions.ParentIonDecoyMassDa; } bool success; // Determine if this was an MS/MS scan // If yes, determine the scan number of the survey scan if (thermoScanInfo.MSLevel <= 1) { // Survey Scan success = ExtractXcaliburSurveyScan(xcaliburAccessor, scanList, spectraCache, dataOutputHandler, sicOptions, thermoScanInfo); } else { // Fragmentation Scan success = ExtractXcaliburFragmentationScan(xcaliburAccessor, scanList, spectraCache, dataOutputHandler, sicOptions, mOptions.BinningOptions, thermoScanInfo); } return(success); }
private void SaveRawDataToDiskWork( StreamWriter dataWriter, StreamWriter scanInfoWriter, clsScanInfo currentScan, clsSpectraCache spectraCache, string inputFileName, bool fragmentationScan, ref int spectrumExportCount) { switch (mOptions.RawDataExportOptions.FileFormat) { case clsRawDataExportOptions.eExportRawDataFileFormatConstants.PEKFile: SavePEKFileToDiskWork(dataWriter, currentScan, spectraCache, inputFileName, fragmentationScan, ref spectrumExportCount); break; case clsRawDataExportOptions.eExportRawDataFileFormatConstants.CSVFile: SaveCSVFilesToDiskWork(dataWriter, scanInfoWriter, currentScan, spectraCache, fragmentationScan, ref spectrumExportCount); break; default: // Unknown format // This code should never be reached break; } }
private void SaveDecon2LSChromatogram( ICollection <clsScanInfo> scanList, clsSpectraCache spectraCache, string outputFilePath) { var scansWritten = 0; var lastStatus = DateTime.UtcNow; using (var writer = new StreamWriter(outputFilePath)) { // Write the file headers WriteDecon2LSScanFileHeaders(writer); // Step through the scans and write each one foreach (var scanItem in scanList) { WriteDecon2LSScanFileEntry(writer, scanItem, spectraCache); if (scansWritten % 250 == 0) { UpdateCacheStats(spectraCache); if (DateTime.UtcNow.Subtract(lastStatus).TotalSeconds >= 30) { lastStatus = DateTime.UtcNow; ReportMessage(string.Format(" {0} / {1} scans processed", scansWritten, scanList.Count)); } } scansWritten += 1; } } }
private void WriteDecon2LSScanFileEntry( StreamWriter writer, clsScanInfo currentScan, clsSpectraCache spectraCache) { int numPeaks; if (spectraCache == null) { numPeaks = 0; } else { if (!spectraCache.GetSpectrum(currentScan.ScanNumber, out var spectrum, true)) { SetLocalErrorCode(clsMASIC.eMasicErrorCodes.ErrorUncachingSpectrum); return; } numPeaks = spectrum.IonCount; } var scanNumber = currentScan.ScanNumber; var msLevel = currentScan.FragScanInfo.MSLevel; if (msLevel < 1) { msLevel = 1; } var numIsotopicSignatures = 0; WriteDecon2LSScanFileEntry(writer, currentScan, scanNumber, msLevel, numPeaks, numIsotopicSignatures); }
public bool SaveBPIs( clsScanList scanList, clsSpectraCache spectraCache, string inputFilePathFull, string outputDirectoryPath) { // This function creates an ICR-2LS compatible .TIC file (using only the MS1 scans), plus // two Decon2LS compatible .CSV files (one for the MS1 scans and one for the MS2, MS3, etc. scans) // Note: Note that SaveExtendedScanStatsFiles() creates a tab-delimited text file with the BPI and TIC information for each scan var currentFilePath = "_MS_scans.csv"; try { var bpiStepCount = 3; UpdateProgress(0, "Saving chromatograms to disk"); var inputFileName = Path.GetFileName(inputFilePathFull); // Disabled in April 2015 since not used // ' First, write a true TIC file (in ICR-2LS format) // outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, eOutputFileTypeConstants.ICRToolsTICChromatogramByScan) // LogMessage("Saving ICR Tools TIC to " + Path.GetFileName(outputFilePath)) // SaveICRToolsChromatogramByScan(scanList.SurveyScans, scanList.SurveyScans.Count, outputFilePath, False, True, inputFilePathFull) var stepsCompleted = 1; UpdateProgress((short)(stepsCompleted / (double)bpiStepCount * 100)); // Second, write an MS-based _scans.csv file (readable with Decon2LS) var msScansFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsMSChromatogramFile); currentFilePath = string.Copy(msScansFilePath); ReportMessage("Saving Decon2LS MS Chromatogram File to " + Path.GetFileName(msScansFilePath)); SaveDecon2LSChromatogram(scanList.SurveyScans, spectraCache, msScansFilePath); stepsCompleted += 1; UpdateProgress((short)(stepsCompleted / (double)bpiStepCount * 100)); // Third, write an MSMS-based _scans.csv file (readable with Decon2LS) var msmsScansFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsMSMSChromatogramFile); currentFilePath = string.Copy(msmsScansFilePath); ReportMessage("Saving Decon2LS MSMS Chromatogram File to " + Path.GetFileName(msmsScansFilePath)); SaveDecon2LSChromatogram(scanList.FragScans, spectraCache, msmsScansFilePath); UpdateProgress(100); return(true); } catch (Exception ex) { ReportError("Error writing the BPI to: " + currentFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } }
public bool XMLOutputFileFinalize( clsDataOutput dataOutputHandler, clsScanList scanList, clsSpectraCache spectraCache, clsProcessingStats processingStats, float processingTimeSec) { var writer = dataOutputHandler.OutputFileHandles.XMLFileForSICs; if (writer == null) { return(false); } try { writer.WriteStartElement("ProcessingStats"); writer.WriteElementString("CacheEventCount", spectraCache.CacheEventCount.ToString()); writer.WriteElementString("UnCacheEventCount", spectraCache.UnCacheEventCount.ToString()); writer.WriteElementString("SpectraPoolHitEventCount", spectraCache.SpectraPoolHitEventCount.ToString()); writer.WriteElementString("PeakMemoryUsageMB", StringUtilities.DblToString(processingStats.PeakMemoryUsageMB, 1)); var effectiveSeconds = processingTimeSec - processingStats.TotalProcessingTimeAtStart; writer.WriteElementString("TotalProcessingTimeSeconds", StringUtilities.DblToString(effectiveSeconds, 1)); writer.WriteEndElement(); if (scanList.ProcessingIncomplete) { writer.WriteElementString("ProcessingComplete", "False"); } else { writer.WriteElementString("ProcessingComplete", "True"); } writer.WriteEndElement(); // Close out the <SICData> element writer.WriteEndDocument(); writer.Close(); } catch (Exception ex) { ReportError("Error finalizing the XML output file", ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } return(true); }
private bool ExtractScanInfoCheckRange( XRawFileIO xcaliburAccessor, ThermoRawFileReader.clsScanInfo thermoScanInfo, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, double percentComplete) { bool success; if (mScanTracking.CheckScanInRange(thermoScanInfo.ScanNumber, thermoScanInfo.RetentionTime, mOptions.SICOptions)) { success = ExtractScanInfoWork(xcaliburAccessor, scanList, spectraCache, dataOutputHandler, mOptions.SICOptions, thermoScanInfo); } else { mScansOutOfRange += 1; success = true; } UpdateProgress((short)Math.Round(percentComplete, 0)); UpdateCacheStats(spectraCache); if (mOptions.AbortProcessing) { scanList.ProcessingIncomplete = true; return(false); } if (DateTime.UtcNow.Subtract(mLastLogTime).TotalSeconds >= 10 || thermoScanInfo.ScanNumber % 500 == 0 && ( thermoScanInfo.ScanNumber >= mOptions.SICOptions.ScanRangeStart && thermoScanInfo.ScanNumber <= mOptions.SICOptions.ScanRangeEnd)) { ReportMessage("Reading scan: " + thermoScanInfo.ScanNumber.ToString()); Console.Write("."); mLastLogTime = DateTime.UtcNow; } return(success); }
public bool ExtractScanInfoFromMGFandCDF( string filePath, clsScanList scanList, clsSpectraCache spectraCache, DataOutput.clsDataOutput dataOutputHandler, bool keepRawSpectra, bool keepMSMSSpectra) { // Returns True if Success, False if failure // Note: This function assumes filePath exists // // This function can be used to read a pair of MGF and NetCDF files that contain MS/MS and MS-only parent ion scans, respectively // Typically, this will apply to LC-MS/MS analyses acquired using an Agilent mass spectrometer running DataAnalysis software // filePath can contain the path to the MGF or to the CDF file; the extension will be removed in order to determine the base file name, // then the two files will be looked for separately var scanTime = 0.0; var cdfReader = new NetCDFReader.clsMSNetCdf(); var mgfReader = new MSDataFileReader.clsMGFFileReader(); try { Console.Write("Reading CDF/MGF data files "); ReportMessage("Reading CDF/MGF data files"); UpdateProgress(0, "Opening data file: " + Environment.NewLine + Path.GetFileName(filePath)); // Obtain the full path to the file var mgfFileInfo = new FileInfo(filePath); var mgfInputFilePathFull = mgfFileInfo.FullName; // Make sure the extension for mgfInputFilePathFull is .MGF mgfInputFilePathFull = Path.ChangeExtension(mgfInputFilePathFull, AGILENT_MSMS_FILE_EXTENSION); var cdfInputFilePathFull = Path.ChangeExtension(mgfInputFilePathFull, AGILENT_MS_FILE_EXTENSION); var datasetID = mOptions.SICOptions.DatasetID; var sicOptions = mOptions.SICOptions; var success = UpdateDatasetFileStats(mgfFileInfo, datasetID); mDatasetFileInfo.ScanCount = 0; // Open a handle to each data file if (!cdfReader.OpenMSCdfFile(cdfInputFilePathFull)) { ReportError("Error opening input data file: " + cdfInputFilePathFull); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } if (!mgfReader.OpenFile(mgfInputFilePathFull)) { ReportError("Error opening input data file: " + mgfInputFilePathFull); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } var msScanCount = cdfReader.GetScanCount(); mDatasetFileInfo.ScanCount = msScanCount; if (msScanCount <= 0) { // No scans found ReportError("No scans found in the input file: " + cdfInputFilePathFull); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } // Reserve memory for all of the Survey Scan data scanList.Initialize(); var scanCount = mOptions.SICOptions.ScanRangeCount; if (scanCount <= 0) { scanCount = msScanCount * 3; } scanList.ReserveListCapacity(scanCount, Math.Min(msScanCount, scanCount / 3)); // Assume there are 2 frag scans for every survey scan mScanTracking.ReserveListCapacity(scanCount); spectraCache.SpectrumCount = Math.Max(spectraCache.SpectrumCount, scanCount); UpdateProgress("Reading CDF/MGF data (" + msScanCount.ToString() + " scans)" + Environment.NewLine + Path.GetFileName(filePath)); ReportMessage("Reading CDF/MGF data; Total MS scan count: " + msScanCount.ToString()); // Read all of the Survey scans from the CDF file // CDF files created by the Agilent XCT list the first scan number as 0; use scanNumberCorrection to correct for this var scanNumberCorrection = 0; for (var msScanIndex = 0; msScanIndex < msScanCount; msScanIndex++) { success = cdfReader.GetScanInfo(msScanIndex, out var scanNumber, out var scanTotalIntensity, out scanTime, out _, out _); if (msScanIndex == 0 && scanNumber == 0) { scanNumberCorrection = 1; } if (!success) { // Error reading CDF file ReportError("Error obtaining data from CDF file: " + cdfInputFilePathFull); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileDataReadError); return(false); } if (scanNumberCorrection > 0) { scanNumber += scanNumberCorrection; } if (mScanTracking.CheckScanInRange(scanNumber, scanTime, sicOptions)) { var newSurveyScan = new clsScanInfo { ScanNumber = scanNumber, TotalIonIntensity = (float)scanTotalIntensity, // Copy the Total Scan Intensity to .TotalIonIntensity ScanHeaderText = string.Empty, ScanTypeName = "MS", }; if (mOptions.CDFTimeInSeconds) { newSurveyScan.ScanTime = (float)(scanTime / 60); } else { newSurveyScan.ScanTime = (float)scanTime; } // Survey scans typically lead to multiple parent ions; we do not record them here newSurveyScan.FragScanInfo.ParentIonInfoIndex = -1; scanList.SurveyScans.Add(newSurveyScan); success = cdfReader.GetMassSpectrum(msScanIndex, out var mzData, out var intensityData, out var intIonCount, out _); if (success && intIonCount > 0) { var msSpectrum = new clsMSSpectrum(scanNumber, mzData, intensityData, intIonCount); double msDataResolution; newSurveyScan.IonCount = msSpectrum.IonCount; newSurveyScan.IonCountRaw = newSurveyScan.IonCount; // Find the base peak ion mass and intensity newSurveyScan.BasePeakIonMZ = FindBasePeakIon(msSpectrum.IonsMZ, msSpectrum.IonsIntensity, out var basePeakIonIntensity, out var mzMin, out var mzMax); newSurveyScan.BasePeakIonIntensity = basePeakIonIntensity; // Determine the minimum positive intensity in this scan newSurveyScan.MinimumPositiveIntensity = mPeakFinder.FindMinimumPositiveValue(msSpectrum.IonsIntensity, 0); if (sicOptions.SICToleranceIsPPM) { // Define MSDataResolution based on the tolerance value that will be used at the lowest m/z in this spectrum, divided by COMPRESS_TOLERANCE_DIVISOR // However, if the lowest m/z value is < 100, then use 100 m/z if (mzMin < 100) { msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, 100) / sicOptions.CompressToleranceDivisorForPPM; } else { msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, mzMin) / sicOptions.CompressToleranceDivisorForPPM; } } else { msDataResolution = sicOptions.SICTolerance / sicOptions.CompressToleranceDivisorForDa; } mScanTracking.ProcessAndStoreSpectrum( newSurveyScan, this, spectraCache, msSpectrum, sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions, clsMASIC.DISCARD_LOW_INTENSITY_MS_DATA_ON_LOAD, sicOptions.CompressMSSpectraData, msDataResolution, keepRawSpectra); } else { newSurveyScan.IonCount = 0; newSurveyScan.IonCountRaw = 0; } // Note: Since we're reading all of the Survey Scan data, we cannot update .MasterScanOrder() at this time } // Note: We need to take msScanCount * 2 since we have to read two different files if (msScanCount > 1) { UpdateProgress((short)(msScanIndex / (double)(msScanCount * 2 - 1) * 100)); } else { UpdateProgress(0); } UpdateCacheStats(spectraCache); if (mOptions.AbortProcessing) { scanList.ProcessingIncomplete = true; break; } if (msScanIndex % 100 == 0) { ReportMessage("Reading MS scan index: " + msScanIndex.ToString()); Console.Write("."); } } // Record the current memory usage (before we close the .CDF file) OnUpdateMemoryUsage(); cdfReader.CloseMSCdfFile(); // We loaded all of the survey scan data above // We can now initialize .MasterScanOrder() var lastSurveyScanIndex = 0; scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, lastSurveyScanIndex); var surveyScansRecorded = new SortedSet <int>() { lastSurveyScanIndex }; // Reset scanNumberCorrection; we might also apply it to MS/MS data scanNumberCorrection = 0; // Now read the MS/MS data from the MGF file do { var fragScanFound = mgfReader.ReadNextSpectrum(out var spectrumInfo); if (!fragScanFound) { break; } mDatasetFileInfo.ScanCount += 1; while (spectrumInfo.ScanNumber < scanList.SurveyScans[lastSurveyScanIndex].ScanNumber) { // The scan number for the current MS/MS spectrum is less than the last survey scan index scan number // This can happen, due to oddities with combining scans when creating the .MGF file // Need to decrement lastSurveyScanIndex until we find the appropriate survey scan lastSurveyScanIndex -= 1; if (lastSurveyScanIndex == 0) { break; } } if (scanNumberCorrection == 0) { // See if udtSpectrumHeaderInfo.ScanNumberStart is equivalent to one of the survey scan numbers, yielding conflicting scan numbers // If it is, then there is an indexing error in the .MGF file; this error was present in .MGF files generated with // an older version of Agilent Chemstation. These files typically have lines like ###MSMS: #13-29 instead of ###MSMS: #13/29/ // If this indexing error is found, then we'll set scanNumberCorrection = 1 and apply it to all subsequent MS/MS scans; // we'll also need to correct prior MS/MS scans for (var surveyScanIndex = lastSurveyScanIndex; surveyScanIndex < scanList.SurveyScans.Count; surveyScanIndex++) { if (scanList.SurveyScans[surveyScanIndex].ScanNumber == spectrumInfo.ScanNumber) { // Conflicting scan numbers were found scanNumberCorrection = 1; // Need to update prior MS/MS scans foreach (var fragScan in scanList.FragScans) { fragScan.ScanNumber += scanNumberCorrection; var scanTimeInterpolated = InterpolateRTandFragScanNumber( scanList.SurveyScans, 0, fragScan.ScanNumber, out var fragScanIterationOut); fragScan.FragScanInfo.FragScanNumber = fragScanIterationOut; fragScan.ScanTime = scanTimeInterpolated; } break; } if (scanList.SurveyScans[surveyScanIndex].ScanNumber > spectrumInfo.ScanNumber) { break; } } } if (scanNumberCorrection > 0) { spectrumInfo.ScanNumber += scanNumberCorrection; spectrumInfo.ScanNumberEnd += scanNumberCorrection; } scanTime = InterpolateRTandFragScanNumber( scanList.SurveyScans, lastSurveyScanIndex, spectrumInfo.ScanNumber, out var fragScanIteration); // Make sure this fragmentation scan isn't present yet in scanList.FragScans // This can occur in Agilent .MGF files if the scan is listed both singly and grouped with other MS/MS scans var validFragScan = true; foreach (var fragScan in scanList.FragScans) { if (fragScan.ScanNumber == spectrumInfo.ScanNumber) { // Duplicate found validFragScan = false; break; } } if (!(validFragScan && mScanTracking.CheckScanInRange(spectrumInfo.ScanNumber, scanTime, sicOptions))) { continue; } // See if lastSurveyScanIndex needs to be updated // At the same time, populate .MasterScanOrder while (lastSurveyScanIndex < scanList.SurveyScans.Count - 1 && spectrumInfo.ScanNumber > scanList.SurveyScans[lastSurveyScanIndex + 1].ScanNumber) { lastSurveyScanIndex += 1; // Add the given SurveyScan to .MasterScanOrder, though only if it hasn't yet been added if (!surveyScansRecorded.Contains(lastSurveyScanIndex)) { surveyScansRecorded.Add(lastSurveyScanIndex); scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, lastSurveyScanIndex); } } scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.FragScan, scanList.FragScans.Count, spectrumInfo.ScanNumber, (float)scanTime); var newFragScan = new clsScanInfo { ScanNumber = spectrumInfo.ScanNumber, ScanTime = (float)scanTime, ScanHeaderText = string.Empty, ScanTypeName = "MSn", }; newFragScan.FragScanInfo.FragScanNumber = fragScanIteration; newFragScan.FragScanInfo.MSLevel = 2; newFragScan.MRMScanInfo.MRMMassCount = 0; scanList.FragScans.Add(newFragScan); var msSpectrum = new clsMSSpectrum(newFragScan.ScanNumber, spectrumInfo.MZList, spectrumInfo.IntensityList, spectrumInfo.DataCount); if (spectrumInfo.DataCount > 0) { newFragScan.IonCount = msSpectrum.IonCount; newFragScan.IonCountRaw = newFragScan.IonCount; // Find the base peak ion mass and intensity newFragScan.BasePeakIonMZ = FindBasePeakIon(msSpectrum.IonsMZ, msSpectrum.IonsIntensity, out var basePeakIonIntensity, out _, out _); newFragScan.BasePeakIonIntensity = basePeakIonIntensity; // Compute the total scan intensity newFragScan.TotalIonIntensity = 0; for (var ionIndex = 0; ionIndex < newFragScan.IonCount; ionIndex++) { newFragScan.TotalIonIntensity += msSpectrum.IonsIntensity[ionIndex]; } // Determine the minimum positive intensity in this scan newFragScan.MinimumPositiveIntensity = mPeakFinder.FindMinimumPositiveValue(msSpectrum.IonsIntensity, 0); var msDataResolution = mOptions.BinningOptions.BinSize / sicOptions.CompressToleranceDivisorForDa; var keepRawSpectrum = keepRawSpectra && keepMSMSSpectra; mScanTracking.ProcessAndStoreSpectrum( newFragScan, this, spectraCache, msSpectrum, sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions, clsMASIC.DISCARD_LOW_INTENSITY_MSMS_DATA_ON_LOAD, sicOptions.CompressMSMSSpectraData, msDataResolution, keepRawSpectrum); } else { newFragScan.IonCount = 0; newFragScan.IonCountRaw = 0; newFragScan.TotalIonIntensity = 0; } mParentIonProcessor.AddUpdateParentIons(scanList, lastSurveyScanIndex, spectrumInfo.ParentIonMZ, scanList.FragScans.Count - 1, spectraCache, sicOptions); // Note: We need to take msScanCount * 2, in addition to adding msScanCount to lastSurveyScanIndex, since we have to read two different files if (msScanCount > 1) { UpdateProgress((short)((lastSurveyScanIndex + msScanCount) / (double)(msScanCount * 2 - 1) * 100)); } else { UpdateProgress(0); } UpdateCacheStats(spectraCache); if (mOptions.AbortProcessing) { scanList.ProcessingIncomplete = true; break; } if (scanList.FragScans.Count % 100 == 0) { ReportMessage("Reading MSMS scan index: " + scanList.FragScans.Count); Console.Write("."); } }while (true); // Record the current memory usage (before we close the .MGF file) OnUpdateMemoryUsage(); mgfReader.CloseFile(); // Check for any other survey scans that need to be added to MasterScanOrder // See if lastSurveyScanIndex needs to be updated // At the same time, populate .MasterScanOrder while (lastSurveyScanIndex < scanList.SurveyScans.Count - 1) { lastSurveyScanIndex += 1; // Note that scanTime is the scan time of the most recent survey scan processed in the above Do loop, so it's not accurate if (mScanTracking.CheckScanInRange(scanList.SurveyScans[lastSurveyScanIndex].ScanNumber, scanTime, sicOptions)) { // Add the given SurveyScan to .MasterScanOrder, though only if it hasn't yet been added if (!surveyScansRecorded.Contains(lastSurveyScanIndex)) { surveyScansRecorded.Add(lastSurveyScanIndex); scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, lastSurveyScanIndex); } } } // Make sure that MasterScanOrder really is sorted by scan number ValidateMasterScanOrderSorting(scanList); // Now that all of the data has been read, write out to the scan stats file, in order of scan number for (var scanIndex = 0; scanIndex < scanList.MasterScanOrderCount; scanIndex++) { var eScanType = scanList.MasterScanOrder[scanIndex].ScanType; clsScanInfo currentScan; if (eScanType == clsScanList.eScanTypeConstants.SurveyScan) { // Survey scan currentScan = scanList.SurveyScans[scanList.MasterScanOrder[scanIndex].ScanIndexPointer]; } else { // Frag Scan currentScan = scanList.FragScans[scanList.MasterScanOrder[scanIndex].ScanIndexPointer]; } SaveScanStatEntry(dataOutputHandler.OutputFileHandles.ScanStats, eScanType, currentScan, sicOptions.DatasetID); } Console.WriteLine(); scanList.SetListCapacityToCount(); mScanTracking.SetListCapacityToCount(); return(success); } catch (Exception ex) { ReportError("Error in ExtractScanInfoFromMGFandCDF", ex, clsMASIC.eMasicErrorCodes.InputFileDataReadError); return(false); } }
private void SavePEKFileToDiskWork( TextWriter writer, clsScanInfo currentScan, clsSpectraCache spectraCache, string inputFileName, bool fragmentationScan, ref int spectrumExportCount) { var exportCount = 0; if (!spectraCache.GetSpectrum(currentScan.ScanNumber, out var spectrum, true)) { SetLocalErrorCode(clsMASIC.eMasicErrorCodes.ErrorUncachingSpectrum); return; } spectrumExportCount += 1; writer.WriteLine("Time domain signal level:" + "\t" + currentScan.BasePeakIonIntensity.ToString("0.000")); // Store the base peak ion intensity as the time domain signal level value writer.WriteLine("MASIC " + mOptions.MASICVersion); // Software version var dataLine = "MS/MS-based PEK file"; if (mOptions.RawDataExportOptions.IncludeMSMS) { dataLine += " (includes both survey scans and fragmentation spectra)"; } else { dataLine += " (includes only survey scans)"; } writer.WriteLine(dataLine); int scanNumber; if (mOptions.RawDataExportOptions.RenumberScans) { scanNumber = spectrumExportCount; } else { scanNumber = currentScan.ScanNumber; } dataLine = "Filename: " + inputFileName + "." + scanNumber.ToString("00000"); writer.WriteLine(dataLine); if (fragmentationScan) { writer.WriteLine("ScanType: Fragmentation Scan"); } else { writer.WriteLine("ScanType: Survey Scan"); } writer.WriteLine("Charge state mass transform results:"); writer.WriteLine("First CS, Number of CS, Abundance, Mass, Standard deviation"); if (spectrum.IonCount > 0) { // Populate intensities and pointerArray() var intensities = new double[spectrum.IonCount]; var pointerArray = new int[spectrum.IonCount]; for (var ionIndex = 0; ionIndex < spectrum.IonCount; ionIndex++) { intensities[ionIndex] = spectrum.IonsIntensity[ionIndex]; pointerArray[ionIndex] = ionIndex; } // Sort pointerArray() based on the intensities in intensities Array.Sort(intensities, pointerArray); int startIndex; if (mOptions.RawDataExportOptions.MaxIonCountPerScan > 0) { // Possibly limit the number of ions to maxIonCount startIndex = spectrum.IonCount - mOptions.RawDataExportOptions.MaxIonCountPerScan; if (startIndex < 0) { startIndex = 0; } } else { startIndex = 0; } // Define the minimum data point intensity value var minimumIntensityCurrentScan = spectrum.IonsIntensity[pointerArray[startIndex]]; // Update the minimum intensity if a higher minimum intensity is defined in .IntensityMinimum minimumIntensityCurrentScan = Math.Max(minimumIntensityCurrentScan, mOptions.RawDataExportOptions.IntensityMinimum); // If mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio is > 0, then possibly update minimumIntensityCurrentScan if (mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio > 0) { minimumIntensityCurrentScan = Math.Max(minimumIntensityCurrentScan, currentScan.BaselineNoiseStats.NoiseLevel * mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio); } exportCount = 0; for (var ionIndex = 0; ionIndex < spectrum.IonCount; ionIndex++) { if (spectrum.IonsIntensity[ionIndex] >= minimumIntensityCurrentScan) { var dataLine1 = "1" + "\t" + "1" + "\t" + spectrum.IonsIntensity[ionIndex] + "\t" + spectrum.IonsMZ[ionIndex] + "\t" + "0"; writer.WriteLine(dataLine1); exportCount += 1; } } } writer.WriteLine("Number of peaks in spectrum = " + spectrum.IonCount.ToString()); writer.WriteLine("Number of isotopic distributions detected = " + exportCount.ToString()); writer.WriteLine(); }
public bool ExportRawDataToDisk( clsScanList scanList, clsSpectraCache spectraCache, string inputFileName, string outputDirectoryPath) { var outputFilePath = "??"; try { StreamWriter dataWriter; StreamWriter scanInfoWriter; switch (mOptions.RawDataExportOptions.FileFormat) { case clsRawDataExportOptions.eExportRawDataFileFormatConstants.PEKFile: outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.PEKFile); dataWriter = new StreamWriter(outputFilePath); scanInfoWriter = null; break; case clsRawDataExportOptions.eExportRawDataFileFormatConstants.CSVFile: outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsIsosFile); var outputFilePath2 = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsScansFile); dataWriter = new StreamWriter(outputFilePath); scanInfoWriter = new StreamWriter(outputFilePath2); // Write the file headers mBPIWriter.WriteDecon2LSIsosFileHeaders(dataWriter); mBPIWriter.WriteDecon2LSScanFileHeaders(scanInfoWriter); break; default: // Unknown format ReportError("Unknown raw data file format: " + mOptions.RawDataExportOptions.FileFormat.ToString()); return(false); } var spectrumExportCount = 0; if (!mOptions.RawDataExportOptions.IncludeMSMS && mOptions.RawDataExportOptions.RenumberScans) { mOptions.RawDataExportOptions.RenumberScans = true; } else { mOptions.RawDataExportOptions.RenumberScans = false; } UpdateProgress(0, "Exporting raw data"); for (var masterOrderIndex = 0; masterOrderIndex < scanList.MasterScanOrderCount; masterOrderIndex++) { var scanPointer = scanList.MasterScanOrder[masterOrderIndex].ScanIndexPointer; if (scanList.MasterScanOrder[masterOrderIndex].ScanType == clsScanList.eScanTypeConstants.SurveyScan) { SaveRawDataToDiskWork(dataWriter, scanInfoWriter, scanList.SurveyScans[scanPointer], spectraCache, inputFileName, false, ref spectrumExportCount); } else if (mOptions.RawDataExportOptions.IncludeMSMS || scanList.FragScans[scanPointer].MRMScanType != ThermoRawFileReader.MRMScanTypeConstants.NotMRM) { // Either we're writing out MS/MS data or this is an MRM scan SaveRawDataToDiskWork(dataWriter, scanInfoWriter, scanList.FragScans[scanPointer], spectraCache, inputFileName, true, ref spectrumExportCount); } if (scanList.MasterScanOrderCount > 1) { UpdateProgress((short)(masterOrderIndex / (double)(scanList.MasterScanOrderCount - 1) * 100)); } else { UpdateProgress(0); } UpdateCacheStats(spectraCache); if (mOptions.AbortProcessing) { break; } } dataWriter.Close(); scanInfoWriter?.Close(); return(true); } catch (Exception ex) { ReportError("Error writing the raw spectra data to: " + outputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } }
private void SaveCSVFilesToDiskWork( StreamWriter dataWriter, StreamWriter scanInfoWriter, clsScanInfo currentScan, clsSpectraCache spectraCache, bool fragmentationScan, ref int spectrumExportCount) { int scanNumber; if (!spectraCache.GetSpectrum(currentScan.ScanNumber, out var spectrum, true)) { SetLocalErrorCode(clsMASIC.eMasicErrorCodes.ErrorUncachingSpectrum); return; } spectrumExportCount += 1; // First, write an entry to the "_scans.csv" file if (mOptions.RawDataExportOptions.RenumberScans) { scanNumber = spectrumExportCount; } else { scanNumber = currentScan.ScanNumber; } int msLevel; if (fragmentationScan) { msLevel = currentScan.FragScanInfo.MSLevel; } else { msLevel = 1; } var numIsotopicSignatures = 0; var numPeaks = spectrum.IonCount; var baselineNoiseLevel = currentScan.BaselineNoiseStats.NoiseLevel; if (baselineNoiseLevel < 1) { baselineNoiseLevel = 1; } mBPIWriter.WriteDecon2LSScanFileEntry(scanInfoWriter, currentScan, scanNumber, msLevel, numPeaks, numIsotopicSignatures); // Now write an entry to the "_isos.csv" file if (spectrum.IonCount > 0) { // Populate intensities and pointerArray() var intensities = new double[spectrum.IonCount]; var pointerArray = new int[spectrum.IonCount]; for (var ionIndex = 0; ionIndex < spectrum.IonCount; ionIndex++) { intensities[ionIndex] = spectrum.IonsIntensity[ionIndex]; pointerArray[ionIndex] = ionIndex; } // Sort pointerArray() based on the intensities in intensities Array.Sort(intensities, pointerArray); int startIndex; if (mOptions.RawDataExportOptions.MaxIonCountPerScan > 0) { // Possibly limit the number of ions to maxIonCount startIndex = spectrum.IonCount - mOptions.RawDataExportOptions.MaxIonCountPerScan; if (startIndex < 0) { startIndex = 0; } } else { startIndex = 0; } // Define the minimum data point intensity value var minimumIntensityCurrentScan = spectrum.IonsIntensity[pointerArray[startIndex]]; // Update the minimum intensity if a higher minimum intensity is defined in .IntensityMinimum minimumIntensityCurrentScan = Math.Max(minimumIntensityCurrentScan, mOptions.RawDataExportOptions.IntensityMinimum); // If mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio is > 0, then possibly update minimumIntensityCurrentScan if (mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio > 0) { minimumIntensityCurrentScan = Math.Max(minimumIntensityCurrentScan, currentScan.BaselineNoiseStats.NoiseLevel * mOptions.RawDataExportOptions.MinimumSignalToNoiseRatio); } for (var ionIndex = 0; ionIndex < spectrum.IonCount; ionIndex++) { if (spectrum.IonsIntensity[ionIndex] >= minimumIntensityCurrentScan) { var charge = 1; var isoFit = 0; var mass = clsUtilities.ConvoluteMass(spectrum.IonsMZ[ionIndex], 1, 0); var peakFWHM = 0; var signalToNoise = spectrum.IonsIntensity[ionIndex] / baselineNoiseLevel; var monoisotopicAbu = -10; var monoPlus2Abu = -10; mBPIWriter.WriteDecon2LSIsosFileEntry( dataWriter, scanNumber, charge, spectrum.IonsIntensity[ionIndex], spectrum.IonsMZ[ionIndex], isoFit, mass, mass, mass, peakFWHM, signalToNoise, monoisotopicAbu, monoPlus2Abu); } } } }
public bool XMLOutputFileInitialize( string inputFilePathFull, string outputDirectoryPath, clsDataOutput dataOutputHandler, clsScanList scanList, clsSpectraCache spectraCache, clsSICOptions sicOptions, clsBinningOptions binningOptions) { var xmlOutputFilePath = string.Empty; try { xmlOutputFilePath = clsDataOutput.ConstructOutputFilePath(inputFilePathFull, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.XMLFile); dataOutputHandler.OutputFileHandles.XMLFileForSICs = new XmlTextWriter(xmlOutputFilePath, System.Text.Encoding.UTF8); var writer = dataOutputHandler.OutputFileHandles.XMLFileForSICs; writer.Formatting = Formatting.Indented; writer.Indentation = 1; writer.WriteStartDocument(true); writer.WriteStartElement("SICData"); writer.WriteStartElement("ProcessingSummary"); writer.WriteElementString("DatasetID", sicOptions.DatasetID.ToString()); writer.WriteElementString("SourceFilePath", inputFilePathFull); string lastModTimeText; string fileSizeBytes; try { var inputFileInfo = new FileInfo(inputFilePathFull); var lastModTime = inputFileInfo.LastWriteTime; lastModTimeText = lastModTime.ToShortDateString() + " " + lastModTime.ToShortTimeString(); fileSizeBytes = inputFileInfo.Length.ToString(); } catch (Exception ex) { lastModTimeText = string.Empty; fileSizeBytes = "0"; } writer.WriteElementString("SourceFileDateTime", lastModTimeText); writer.WriteElementString("SourceFileSizeBytes", fileSizeBytes); writer.WriteElementString("MASICProcessingDate", DateTime.Now.ToString(clsDatasetStatsSummarizer.DATE_TIME_FORMAT_STRING)); writer.WriteElementString("MASICVersion", mOptions.MASICVersion); writer.WriteElementString("MASICPeakFinderDllVersion", mOptions.PeakFinderVersion); writer.WriteElementString("ScanCountTotal", scanList.MasterScanOrderCount.ToString()); writer.WriteElementString("SurveyScanCount", scanList.SurveyScans.Count.ToString()); writer.WriteElementString("FragScanCount", scanList.FragScans.Count.ToString()); writer.WriteElementString("SkipMSMSProcessing", mOptions.SkipMSMSProcessing.ToString()); writer.WriteElementString("ParentIonDecoyMassDa", mOptions.ParentIonDecoyMassDa.ToString("0.0000")); writer.WriteEndElement(); writer.WriteStartElement("MemoryOptions"); writer.WriteElementString("CacheAlwaysDisabled", spectraCache.DiskCachingAlwaysDisabled.ToString()); writer.WriteElementString("CacheSpectraToRetainInMemory", spectraCache.CacheSpectraToRetainInMemory.ToString()); writer.WriteEndElement(); writer.WriteStartElement("SICOptions"); // SIC Options // "SICToleranceDa" is a legacy parameter; If the SIC tolerance is in PPM, then "SICToleranceDa" is the Da tolerance at 1000 m/z writer.WriteElementString("SICToleranceDa", clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, 1000).ToString("0.0000")); writer.WriteElementString("SICTolerance", sicOptions.SICTolerance.ToString("0.0000")); writer.WriteElementString("SICToleranceIsPPM", sicOptions.SICToleranceIsPPM.ToString()); writer.WriteElementString("RefineReportedParentIonMZ", sicOptions.RefineReportedParentIonMZ.ToString()); writer.WriteElementString("ScanRangeStart", sicOptions.ScanRangeStart.ToString()); writer.WriteElementString("ScanRangeEnd", sicOptions.ScanRangeEnd.ToString()); writer.WriteElementString("RTRangeStart", sicOptions.RTRangeStart.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("RTRangeEnd", sicOptions.RTRangeEnd.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("CompressMSSpectraData", sicOptions.CompressMSSpectraData.ToString()); writer.WriteElementString("CompressMSMSSpectraData", sicOptions.CompressMSMSSpectraData.ToString()); writer.WriteElementString("CompressToleranceDivisorForDa", sicOptions.CompressToleranceDivisorForDa.ToString("0.0")); writer.WriteElementString("CompressToleranceDivisorForPPM", sicOptions.CompressToleranceDivisorForPPM.ToString("0.0")); writer.WriteElementString("MaxSICPeakWidthMinutesBackward", sicOptions.MaxSICPeakWidthMinutesBackward.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("MaxSICPeakWidthMinutesForward", sicOptions.MaxSICPeakWidthMinutesForward.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("IntensityThresholdFractionMax", StringUtilities.DblToString(sicOptions.SICPeakFinderOptions.IntensityThresholdFractionMax, 5)); writer.WriteElementString("IntensityThresholdAbsoluteMinimum", sicOptions.SICPeakFinderOptions.IntensityThresholdAbsoluteMinimum.ToString(CultureInfo.InvariantCulture)); // Peak Finding Options var baselineNoiseOptions = sicOptions.SICPeakFinderOptions.SICBaselineNoiseOptions; writer.WriteElementString("SICNoiseThresholdMode", baselineNoiseOptions.BaselineNoiseMode.ToString()); writer.WriteElementString("SICNoiseThresholdIntensity", baselineNoiseOptions.BaselineNoiseLevelAbsolute.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("SICNoiseFractionLowIntensityDataToAverage", StringUtilities.DblToString(baselineNoiseOptions.TrimmedMeanFractionLowIntensityDataToAverage, 5)); writer.WriteElementString("SICNoiseMinimumSignalToNoiseRatio", baselineNoiseOptions.MinimumSignalToNoiseRatio.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("MaxDistanceScansNoOverlap", sicOptions.SICPeakFinderOptions.MaxDistanceScansNoOverlap.ToString()); writer.WriteElementString("MaxAllowedUpwardSpikeFractionMax", StringUtilities.DblToString(sicOptions.SICPeakFinderOptions.MaxAllowedUpwardSpikeFractionMax, 5)); writer.WriteElementString("InitialPeakWidthScansScaler", sicOptions.SICPeakFinderOptions.InitialPeakWidthScansScaler.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("InitialPeakWidthScansMaximum", sicOptions.SICPeakFinderOptions.InitialPeakWidthScansMaximum.ToString()); writer.WriteElementString("FindPeaksOnSmoothedData", sicOptions.SICPeakFinderOptions.FindPeaksOnSmoothedData.ToString()); writer.WriteElementString("SmoothDataRegardlessOfMinimumPeakWidth", sicOptions.SICPeakFinderOptions.SmoothDataRegardlessOfMinimumPeakWidth.ToString()); writer.WriteElementString("UseButterworthSmooth", sicOptions.SICPeakFinderOptions.UseButterworthSmooth.ToString()); writer.WriteElementString("ButterworthSamplingFrequency", StringUtilities.DblToString(sicOptions.SICPeakFinderOptions.ButterworthSamplingFrequency, 5)); writer.WriteElementString("ButterworthSamplingFrequencyDoubledForSIMData", sicOptions.SICPeakFinderOptions.ButterworthSamplingFrequencyDoubledForSIMData.ToString()); writer.WriteElementString("UseSavitzkyGolaySmooth", sicOptions.SICPeakFinderOptions.UseSavitzkyGolaySmooth.ToString()); writer.WriteElementString("SavitzkyGolayFilterOrder", sicOptions.SICPeakFinderOptions.SavitzkyGolayFilterOrder.ToString()); var noiseThresholdOptions = sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions; writer.WriteElementString("MassSpectraNoiseThresholdMode", noiseThresholdOptions.BaselineNoiseMode.ToString()); writer.WriteElementString("MassSpectraNoiseThresholdIntensity", noiseThresholdOptions.BaselineNoiseLevelAbsolute.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("MassSpectraNoiseFractionLowIntensityDataToAverage", StringUtilities.DblToString(noiseThresholdOptions.TrimmedMeanFractionLowIntensityDataToAverage, 5)); writer.WriteElementString("MassSpectraNoiseMinimumSignalToNoiseRatio", noiseThresholdOptions.MinimumSignalToNoiseRatio.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("ReplaceSICZeroesWithMinimumPositiveValueFromMSData", sicOptions.ReplaceSICZeroesWithMinimumPositiveValueFromMSData.ToString()); writer.WriteElementString("SaveSmoothedData", sicOptions.SaveSmoothedData.ToString()); // Similarity options writer.WriteElementString("SimilarIonMZToleranceHalfWidth", sicOptions.SimilarIonMZToleranceHalfWidth.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("SimilarIonToleranceHalfWidthMinutes", sicOptions.SimilarIonToleranceHalfWidthMinutes.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("SpectrumSimilarityMinimum", sicOptions.SpectrumSimilarityMinimum.ToString(CultureInfo.InvariantCulture)); writer.WriteEndElement(); writer.WriteStartElement("BinningOptions"); writer.WriteElementString("BinStartX", binningOptions.StartX.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("BinEndX", binningOptions.EndX.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("BinSize", binningOptions.BinSize.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("MaximumBinCount", binningOptions.MaximumBinCount.ToString()); writer.WriteElementString("IntensityPrecisionPercent", binningOptions.IntensityPrecisionPercent.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("Normalize", binningOptions.Normalize.ToString()); writer.WriteElementString("SumAllIntensitiesForBin", binningOptions.SumAllIntensitiesForBin.ToString()); writer.WriteEndElement(); writer.WriteStartElement("CustomSICValues"); writer.WriteElementString("MZList", mOptions.CustomSICList.RawTextMZList); writer.WriteElementString("MZToleranceDaList", CheckForEmptyToleranceList(mOptions.CustomSICList.RawTextMZToleranceDaList)); writer.WriteElementString("ScanCenterList", mOptions.CustomSICList.RawTextScanOrAcqTimeCenterList); writer.WriteElementString("ScanToleranceList", CheckForEmptyToleranceList(mOptions.CustomSICList.RawTextScanOrAcqTimeToleranceList)); writer.WriteElementString("ScanTolerance", mOptions.CustomSICList.ScanOrAcqTimeTolerance.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("ScanType", mOptions.CustomSICList.ScanToleranceType.ToString()); writer.WriteElementString("LimitSearchToCustomMZList", mOptions.CustomSICList.LimitSearchToCustomMZList.ToString()); writer.WriteEndElement(); } catch (Exception ex) { ReportError("Error initializing the XML output file: " + xmlOutputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } return(true); }
private bool LoadSpectraForThermoRawFile( XRawFileIO xcaliburAccessor, clsSpectraCache spectraCache, clsScanInfo scanInfo, clsBaselineNoiseOptions noiseThresholdOptions, bool discardLowIntensityData, bool compressSpectraData, double msDataResolution, bool keepRawSpectrum) { var lastKnownLocation = "Start"; try { // Load the ions for this scan lastKnownLocation = "xcaliburAccessor.GetScanData for scan " + scanInfo.ScanNumber; // Retrieve the m/z and intensity values for the given scan // We retrieve the profile-mode data, since that's required for determining spectrum noise scanInfo.IonCountRaw = xcaliburAccessor.GetScanData(scanInfo.ScanNumber, out var mzList, out var intensityList); if (scanInfo.IonCountRaw > 0) { var ionCountVerified = VerifyDataSorted(scanInfo.ScanNumber, scanInfo.IonCountRaw, mzList, intensityList); if (ionCountVerified != scanInfo.IonCountRaw) { scanInfo.IonCountRaw = ionCountVerified; } } scanInfo.IonCount = scanInfo.IonCountRaw; lastKnownLocation = "Instantiate new clsMSSpectrum"; var msSpectrum = new clsMSSpectrum(scanInfo.ScanNumber, mzList, intensityList, scanInfo.IonCountRaw); lastKnownLocation = "Manually determine the base peak m/z and base peak intensity"; // ReSharper disable once CommentTypo // Regarding BPI, comparison of data read via the ThermoRawFileReader vs. // that read from the .mzML file for dataset QC_Shew_18_02-run1_02Mar19_Arwen_18-11-02 // showed that 25% of the spectra had incorrect BPI values double totalIonIntensity = 0; double basePeakIntensity = 0; double basePeakMz = 0; for (var ionIndex = 0; ionIndex < scanInfo.IonCountRaw; ionIndex++) { totalIonIntensity += intensityList[ionIndex]; if (intensityList[ionIndex] > basePeakIntensity) { basePeakIntensity = intensityList[ionIndex]; basePeakMz = mzList[ionIndex]; } } if (Math.Abs(scanInfo.BasePeakIonMZ - basePeakMz) > 0.1) { mBpiUpdateCount += 1; if (mBpiUpdateCount < 10) { ConsoleMsgUtils.ShowDebug("Updating BPI in scan {0} from {1:F3} m/z to {2:F3} m/z, and BPI Intensity from {3:F0} to {4:F0}", scanInfo.ScanNumber, scanInfo.BasePeakIonMZ, basePeakMz, scanInfo.BasePeakIonIntensity, basePeakIntensity); } scanInfo.BasePeakIonMZ = basePeakMz; scanInfo.BasePeakIonIntensity = basePeakIntensity; } // Determine the minimum positive intensity in this scan lastKnownLocation = "Call mMASICPeakFinder.FindMinimumPositiveValue"; scanInfo.MinimumPositiveIntensity = mPeakFinder.FindMinimumPositiveValue(msSpectrum.IonsIntensity, 0); if (msSpectrum.IonCount > 0) { if (scanInfo.TotalIonIntensity < float.Epsilon) { scanInfo.TotalIonIntensity = totalIonIntensity; } } else { scanInfo.TotalIonIntensity = 0; } bool discardLowIntensityDataWork; bool compressSpectraDataWork; if (scanInfo.MRMScanType == MRMScanTypeConstants.NotMRM) { discardLowIntensityDataWork = discardLowIntensityData; compressSpectraDataWork = compressSpectraData; } else { discardLowIntensityDataWork = false; compressSpectraDataWork = false; } lastKnownLocation = "Call ProcessAndStoreSpectrum"; mScanTracking.ProcessAndStoreSpectrum( scanInfo, this, spectraCache, msSpectrum, noiseThresholdOptions, discardLowIntensityDataWork, compressSpectraDataWork, msDataResolution, keepRawSpectrum); } catch (Exception ex) { ReportError("Error in LoadSpectraForThermoRawFile (LastKnownLocation: " + lastKnownLocation + ")", ex, clsMASIC.eMasicErrorCodes.InputFileDataReadError); return(false); } return(true); }
private bool ExtractXcaliburFragmentationScan( XRawFileIO xcaliburAccessor, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, clsSICOptions sicOptions, clsBinningOptions binningOptions, ThermoRawFileReader.clsScanInfo thermoScanInfo) { // Note that MinimumPositiveIntensity will be determined in LoadSpectraForThermoRawFile var scanInfo = new clsScanInfo(thermoScanInfo.ParentIonMZ) { ScanNumber = thermoScanInfo.ScanNumber, ScanTime = (float)thermoScanInfo.RetentionTime, ScanHeaderText = XRawFileIO.MakeGenericThermoScanFilter(thermoScanInfo.FilterText), ScanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(thermoScanInfo.FilterText), BasePeakIonMZ = thermoScanInfo.BasePeakMZ, BasePeakIonIntensity = thermoScanInfo.BasePeakIntensity, TotalIonIntensity = thermoScanInfo.TotalIonCurrent, MinimumPositiveIntensity = 0, ZoomScan = thermoScanInfo.ZoomScan, SIMScan = thermoScanInfo.SIMScan, MRMScanType = thermoScanInfo.MRMScanType }; // Typically .EventNumber is 1 for the parent-ion scan; 2 for 1st frag scan, 3 for 2nd frag scan, etc. // This resets for each new parent-ion scan scanInfo.FragScanInfo.FragScanNumber = thermoScanInfo.EventNumber - 1; // The .EventNumber value is sometimes wrong; need to check for this // For example, if the dataset only has MS2 scans and no parent-ion scan, .EventNumber will be 2 for every MS2 scan if (scanList.FragScans.Count > 0) { var prevFragScan = scanList.FragScans[scanList.FragScans.Count - 1]; if (prevFragScan.ScanNumber == scanInfo.ScanNumber - 1) { if (scanInfo.FragScanInfo.FragScanNumber <= prevFragScan.FragScanInfo.FragScanNumber) { scanInfo.FragScanInfo.FragScanNumber = prevFragScan.FragScanInfo.FragScanNumber + 1; } } } scanInfo.FragScanInfo.MSLevel = thermoScanInfo.MSLevel; if (scanInfo.MRMScanType != MRMScanTypeConstants.NotMRM) { // This is an MRM scan scanList.MRMDataPresent = true; scanInfo.MRMScanInfo = clsMRMProcessing.DuplicateMRMInfo(thermoScanInfo.MRMInfo, thermoScanInfo.ParentIonMZ); if (scanList.SurveyScans.Count == 0) { // Need to add a "fake" survey scan that we can map this parent ion to mLastNonZoomSurveyScanIndex = scanList.AddFakeSurveyScan(); } } else { scanInfo.MRMScanInfo.MRMMassCount = 0; } scanInfo.LowMass = thermoScanInfo.LowMass; scanInfo.HighMass = thermoScanInfo.HighMass; scanInfo.IsFTMS = thermoScanInfo.IsFTMS; // Store the ScanEvent values in .ExtendedHeaderInfo StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.ScanEvents); // Store the collision mode and possibly the scan filter text scanInfo.FragScanInfo.CollisionMode = thermoScanInfo.CollisionMode; StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_COLLISION_MODE, thermoScanInfo.CollisionMode); if (mOptions.WriteExtendedStatsIncludeScanFilterText) { StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_SCAN_FILTER_TEXT, thermoScanInfo.FilterText); } if (mOptions.WriteExtendedStatsStatusLog) { // Store the StatusLog values in .ExtendedHeaderInfo StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.StatusLog, mOptions.StatusLogKeyNameFilterList); } scanList.FragScans.Add(scanInfo); var fragScanIndex = scanList.FragScans.Count - 1; scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.FragScan, fragScanIndex); // Note: Even if keepRawSpectra = False, we still need to load the raw data so that we can compute the noise level for the spectrum var msDataResolution = binningOptions.BinSize / sicOptions.CompressToleranceDivisorForDa; var success = LoadSpectraForThermoRawFile( xcaliburAccessor, spectraCache, scanInfo, sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions, clsMASIC.DISCARD_LOW_INTENSITY_MSMS_DATA_ON_LOAD, sicOptions.CompressMSMSSpectraData, msDataResolution, mKeepRawSpectra && mKeepMSMSSpectra); if (!success) { return(false); } SaveScanStatEntry(dataOutputHandler.OutputFileHandles.ScanStats, clsScanList.eScanTypeConstants.FragScan, scanInfo, sicOptions.DatasetID); if (thermoScanInfo.MRMScanType == MRMScanTypeConstants.NotMRM) { // This is not an MRM scan mParentIonProcessor.AddUpdateParentIons(scanList, mLastNonZoomSurveyScanIndex, thermoScanInfo.ParentIonMZ, fragScanIndex, spectraCache, sicOptions); } else { // This is an MRM scan mParentIonProcessor.AddUpdateParentIons(scanList, mLastNonZoomSurveyScanIndex, thermoScanInfo.ParentIonMZ, scanInfo.MRMScanInfo, spectraCache, sicOptions); } if (mLastNonZoomSurveyScanIndex >= 0) { var precursorScanNumber = scanList.SurveyScans[mLastNonZoomSurveyScanIndex].ScanNumber; // Compute the interference of the parent ion in the MS1 spectrum for this frag scan scanInfo.FragScanInfo.InterferenceScore = ComputeInterference(xcaliburAccessor, thermoScanInfo, precursorScanNumber); } return(true); }
private bool ExtractXcaliburSurveyScan( XRawFileIO xcaliburAccessor, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, clsSICOptions sicOptions, ThermoRawFileReader.clsScanInfo thermoScanInfo) { var scanInfo = new clsScanInfo() { ScanNumber = thermoScanInfo.ScanNumber, ScanTime = (float)thermoScanInfo.RetentionTime, ScanHeaderText = XRawFileIO.MakeGenericThermoScanFilter(thermoScanInfo.FilterText), ScanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(thermoScanInfo.FilterText), BasePeakIonMZ = thermoScanInfo.BasePeakMZ, BasePeakIonIntensity = thermoScanInfo.BasePeakIntensity, TotalIonIntensity = thermoScanInfo.TotalIonCurrent, MinimumPositiveIntensity = 0, // This will be determined in LoadSpectraForThermoRawFile ZoomScan = thermoScanInfo.ZoomScan, SIMScan = thermoScanInfo.SIMScan, MRMScanType = thermoScanInfo.MRMScanType, LowMass = thermoScanInfo.LowMass, HighMass = thermoScanInfo.HighMass, IsFTMS = thermoScanInfo.IsFTMS }; // Survey scans typically lead to multiple parent ions; we do not record them here scanInfo.FragScanInfo.ParentIonInfoIndex = -1; if (scanInfo.MRMScanType != MRMScanTypeConstants.NotMRM) { // This is an MRM scan scanList.MRMDataPresent = true; } if (scanInfo.SIMScan) { scanList.SIMDataPresent = true; var simKey = scanInfo.LowMass + "_" + scanInfo.HighMass; if (mSIMScanMapping.TryGetValue(simKey, out var simIndex)) { scanInfo.SIMIndex = simIndex; } else { scanInfo.SIMIndex = mSIMScanMapping.Count; mSIMScanMapping.Add(simKey, mSIMScanMapping.Count); } } // Store the ScanEvent values in .ExtendedHeaderInfo StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.ScanEvents); // Store the collision mode and possibly the scan filter text scanInfo.FragScanInfo.CollisionMode = thermoScanInfo.CollisionMode; StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_COLLISION_MODE, thermoScanInfo.CollisionMode); if (mOptions.WriteExtendedStatsIncludeScanFilterText) { StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_SCAN_FILTER_TEXT, thermoScanInfo.FilterText); } if (mOptions.WriteExtendedStatsStatusLog) { // Store the StatusLog values in .ExtendedHeaderInfo StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.StatusLog, mOptions.StatusLogKeyNameFilterList); } scanList.SurveyScans.Add(scanInfo); if (!scanInfo.ZoomScan) { mLastNonZoomSurveyScanIndex = scanList.SurveyScans.Count - 1; } scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, scanList.SurveyScans.Count - 1); double msDataResolution; if (sicOptions.SICToleranceIsPPM) { // Define MSDataResolution based on the tolerance value that will be used at the lowest m/z in this spectrum, divided by sicOptions.CompressToleranceDivisorForPPM // However, if the lowest m/z value is < 100, then use 100 m/z if (thermoScanInfo.LowMass < 100) { msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, 100) / sicOptions.CompressToleranceDivisorForPPM; } else { msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, thermoScanInfo.LowMass) / sicOptions.CompressToleranceDivisorForPPM; } } else { msDataResolution = sicOptions.SICTolerance / sicOptions.CompressToleranceDivisorForDa; } // Note: Even if mKeepRawSpectra = False, we still need to load the raw data so that we can compute the noise level for the spectrum var success = LoadSpectraForThermoRawFile( xcaliburAccessor, spectraCache, scanInfo, sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions, clsMASIC.DISCARD_LOW_INTENSITY_MS_DATA_ON_LOAD, sicOptions.CompressMSSpectraData, msDataResolution, mKeepRawSpectra); if (!success) { return(false); } SaveScanStatEntry(dataOutputHandler.OutputFileHandles.ScanStats, clsScanList.eScanTypeConstants.SurveyScan, scanInfo, sicOptions.DatasetID); return(true); }
/// <summary> /// Read scan data and ions from a Thermo .raw file /// </summary> /// <param name="filePath"></param> /// <param name="scanList"></param> /// <param name="spectraCache"></param> /// <param name="dataOutputHandler"></param> /// <param name="keepRawSpectra"></param> /// <param name="keepMSMSSpectra"></param> /// <returns>True if Success, False if failure</returns> /// <remarks>Assumes filePath exists</remarks> public bool ExtractScanInfoFromXcaliburDataFile( string filePath, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, bool keepRawSpectra, bool keepMSMSSpectra) { // Use XrawFileIO to read the .Raw files (it uses ThermoFisher.CommonCore) var readerOptions = new ThermoReaderOptions { LoadMSMethodInfo = mOptions.WriteMSMethodFile, LoadMSTuneInfo = mOptions.WriteMSTuneFile }; var xcaliburAccessor = new XRawFileIO(readerOptions) { ScanInfoCacheMaxSize = 0 // Don't cache scanInfo objects }; RegisterEvents(xcaliburAccessor); mBpiUpdateCount = 0; // Assume success for now var success = true; try { Console.Write("Reading Thermo .raw file "); ReportMessage("Reading Thermo .raw file"); UpdateProgress(0, "Opening data file:" + Environment.NewLine + Path.GetFileName(filePath)); // Obtain the full path to the file var rawFileInfo = new FileInfo(filePath); var inputFileFullPath = rawFileInfo.FullName; // Open a handle to the data file if (!xcaliburAccessor.OpenRawFile(inputFileFullPath)) { ReportError("Error opening input data file: " + inputFileFullPath + " (xcaliburAccessor.OpenRawFile returned False)"); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } var datasetID = mOptions.SICOptions.DatasetID; success = UpdateDatasetFileStats(rawFileInfo, datasetID, xcaliburAccessor); var metadataWriter = new clsThermoMetadataWriter(); RegisterEvents(metadataWriter); if (mOptions.WriteMSMethodFile) { metadataWriter.SaveMSMethodFile(xcaliburAccessor, dataOutputHandler); } if (mOptions.WriteMSTuneFile) { metadataWriter.SaveMSTuneFile(xcaliburAccessor, dataOutputHandler); } var scanCount = xcaliburAccessor.GetNumScans(); if (scanCount <= 0) { // No scans found ReportError("No scans found in the input file: " + filePath); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } var scanStart = xcaliburAccessor.ScanStart; var scanEnd = xcaliburAccessor.ScanEnd; InitOptions(scanList, keepRawSpectra, keepMSMSSpectra); UpdateProgress(string.Format("Reading Xcalibur data ({0:N0} scans){1}", scanCount, Environment.NewLine + Path.GetFileName(filePath))); ReportMessage(string.Format("Reading Xcalibur data; Total scan count: {0:N0}", scanCount)); var scanCountToRead = scanEnd - scanStart + 1; var scansEst = mOptions.SICOptions.ScanRangeCount; if (scansEst <= 0) { scansEst = scanCountToRead; } scanList.ReserveListCapacity(scansEst); mScanTracking.ReserveListCapacity(scansEst); spectraCache.SpectrumCount = Math.Max(spectraCache.SpectrumCount, scansEst); for (var scanNumber = scanStart; scanNumber <= scanEnd; scanNumber++) { if (!mScanTracking.CheckScanInRange(scanNumber, mOptions.SICOptions)) { mScansOutOfRange += 1; continue; } success = xcaliburAccessor.GetScanInfo(scanNumber, out ThermoRawFileReader.clsScanInfo thermoScanInfo); if (!success) { // GetScanInfo returned false ReportWarning("xcaliburAccessor.GetScanInfo returned false for scan " + scanNumber.ToString() + "; aborting read"); break; } var percentComplete = scanList.MasterScanOrderCount / (double)(scanCountToRead) * 100; var extractSuccess = ExtractScanInfoCheckRange(xcaliburAccessor, thermoScanInfo, scanList, spectraCache, dataOutputHandler, percentComplete); if (!extractSuccess) { break; } } Console.WriteLine(); scanList.SetListCapacityToCount(); mScanTracking.SetListCapacityToCount(); // Shrink the memory usage of the scanList arrays success = FinalizeScanList(scanList, rawFileInfo); } catch (Exception ex) { Console.WriteLine(ex.StackTrace); ReportError("Error in ExtractScanInfoFromXcaliburDataFile", ex, clsMASIC.eMasicErrorCodes.InputFileDataReadError); } // Close the handle to the data file xcaliburAccessor.CloseRawFile(); return(success); }