private bool ExtractScanInfoWork( XRawFileIO xcaliburAccessor, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, clsSICOptions sicOptions, ThermoRawFileReader.clsScanInfo thermoScanInfo) { if (thermoScanInfo.ParentIonMZ > 0 && Math.Abs(mOptions.ParentIonDecoyMassDa) > 0) { thermoScanInfo.ParentIonMZ += mOptions.ParentIonDecoyMassDa; } bool success; // Determine if this was an MS/MS scan // If yes, determine the scan number of the survey scan if (thermoScanInfo.MSLevel <= 1) { // Survey Scan success = ExtractXcaliburSurveyScan(xcaliburAccessor, scanList, spectraCache, dataOutputHandler, sicOptions, thermoScanInfo); } else { // Fragmentation Scan success = ExtractXcaliburFragmentationScan(xcaliburAccessor, scanList, spectraCache, dataOutputHandler, sicOptions, mOptions.BinningOptions, thermoScanInfo); } return(success); }
/// <summary> /// Test ScanOrAcqTimeToAbsolute /// </summary> /// <param name="scanList"></param> /// <param name="scanNumScanConverter"></param> /// <param name="scanNumber">Absolute scan number</param> /// <param name="relativeTime">Relative scan (value between 0 and 1)</param> /// <param name="scanTime">Scan time</param> private void TestScanConversionToAbsolute( clsScanList scanList, clsScanNumScanTimeConversion scanNumScanConverter, KeyValuePair <int, int> scanNumber, KeyValuePair <float, float> relativeTime, KeyValuePair <float, float> scanTime) { try { // Find the scan number corresponding to each of these values float result1 = scanNumScanConverter.ScanOrAcqTimeToAbsolute(scanList, scanNumber.Key, clsCustomSICList.eCustomSICScanTypeConstants.Absolute, false); Console.WriteLine(scanNumber.Key + " -> " + result1); Assert.AreEqual(scanNumber.Value, result1, 1E-05); float result2 = scanNumScanConverter.ScanOrAcqTimeToAbsolute(scanList, relativeTime.Key, clsCustomSICList.eCustomSICScanTypeConstants.Relative, false); Console.WriteLine(relativeTime.Key + " -> " + result2); Assert.AreEqual(relativeTime.Value, result2, 1E-05); float result3 = scanNumScanConverter.ScanOrAcqTimeToAbsolute(scanList, scanTime.Key, clsCustomSICList.eCustomSICScanTypeConstants.AcquisitionTime, false); Console.WriteLine(scanTime.Key + " -> " + result3); Assert.AreEqual(scanTime.Value, result3, 1E-05); Console.WriteLine(); } catch (Exception ex) { Console.WriteLine("Error caught: " + ex.Message); } }
private clsScanInfo GetScanByMasterScanIndex(clsScanList scanList, int masterScanIndex) { var currentScan = new clsScanInfo(); if (scanList.MasterScanOrder != null) { if (masterScanIndex < 0) { masterScanIndex = 0; } else if (masterScanIndex >= scanList.MasterScanOrderCount) { masterScanIndex = scanList.MasterScanOrderCount - 1; } switch (scanList.MasterScanOrder[masterScanIndex].ScanType) { case clsScanList.eScanTypeConstants.SurveyScan: // Survey scan currentScan = scanList.SurveyScans[scanList.MasterScanOrder[masterScanIndex].ScanIndexPointer]; break; case clsScanList.eScanTypeConstants.FragScan: // Frag Scan currentScan = scanList.FragScans[scanList.MasterScanOrder[masterScanIndex].ScanIndexPointer]; break; default: // Unknown scan type break; } } return(currentScan); }
public bool SaveBPIs( clsScanList scanList, clsSpectraCache spectraCache, string inputFilePathFull, string outputDirectoryPath) { // This function creates an ICR-2LS compatible .TIC file (using only the MS1 scans), plus // two Decon2LS compatible .CSV files (one for the MS1 scans and one for the MS2, MS3, etc. scans) // Note: Note that SaveExtendedScanStatsFiles() creates a tab-delimited text file with the BPI and TIC information for each scan var currentFilePath = "_MS_scans.csv"; try { var bpiStepCount = 3; UpdateProgress(0, "Saving chromatograms to disk"); var inputFileName = Path.GetFileName(inputFilePathFull); // Disabled in April 2015 since not used // ' First, write a true TIC file (in ICR-2LS format) // outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, eOutputFileTypeConstants.ICRToolsTICChromatogramByScan) // LogMessage("Saving ICR Tools TIC to " + Path.GetFileName(outputFilePath)) // SaveICRToolsChromatogramByScan(scanList.SurveyScans, scanList.SurveyScans.Count, outputFilePath, False, True, inputFilePathFull) var stepsCompleted = 1; UpdateProgress((short)(stepsCompleted / (double)bpiStepCount * 100)); // Second, write an MS-based _scans.csv file (readable with Decon2LS) var msScansFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsMSChromatogramFile); currentFilePath = string.Copy(msScansFilePath); ReportMessage("Saving Decon2LS MS Chromatogram File to " + Path.GetFileName(msScansFilePath)); SaveDecon2LSChromatogram(scanList.SurveyScans, spectraCache, msScansFilePath); stepsCompleted += 1; UpdateProgress((short)(stepsCompleted / (double)bpiStepCount * 100)); // Third, write an MSMS-based _scans.csv file (readable with Decon2LS) var msmsScansFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsMSMSChromatogramFile); currentFilePath = string.Copy(msmsScansFilePath); ReportMessage("Saving Decon2LS MSMS Chromatogram File to " + Path.GetFileName(msmsScansFilePath)); SaveDecon2LSChromatogram(scanList.FragScans, spectraCache, msmsScansFilePath); UpdateProgress(100); return(true); } catch (Exception ex) { ReportError("Error writing the BPI to: " + currentFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } }
protected void InitBaseOptions(clsScanList scanList, bool keepRawSpectra, bool keepMSMSSpectra) { mLastNonZoomSurveyScanIndex = -1; mScansOutOfRange = 0; scanList.SIMDataPresent = false; scanList.MRMDataPresent = false; mKeepRawSpectra = keepRawSpectra; mKeepMSMSSpectra = keepMSMSSpectra; mLastLogTime = DateTime.UtcNow; }
private void InitOptions(clsScanList scanList, bool keepRawSpectra, bool keepMSMSSpectra) { if (mOptions.SICOptions.ScanRangeStart > 0 && mOptions.SICOptions.ScanRangeEnd == 0) { mOptions.SICOptions.ScanRangeEnd = int.MaxValue; } scanList.Initialize(); mSIMScanMapping.Clear(); InitBaseOptions(scanList, keepRawSpectra, keepMSMSSpectra); }
public bool XMLOutputFileFinalize( clsDataOutput dataOutputHandler, clsScanList scanList, clsSpectraCache spectraCache, clsProcessingStats processingStats, float processingTimeSec) { var writer = dataOutputHandler.OutputFileHandles.XMLFileForSICs; if (writer == null) { return(false); } try { writer.WriteStartElement("ProcessingStats"); writer.WriteElementString("CacheEventCount", spectraCache.CacheEventCount.ToString()); writer.WriteElementString("UnCacheEventCount", spectraCache.UnCacheEventCount.ToString()); writer.WriteElementString("SpectraPoolHitEventCount", spectraCache.SpectraPoolHitEventCount.ToString()); writer.WriteElementString("PeakMemoryUsageMB", StringUtilities.DblToString(processingStats.PeakMemoryUsageMB, 1)); var effectiveSeconds = processingTimeSec - processingStats.TotalProcessingTimeAtStart; writer.WriteElementString("TotalProcessingTimeSeconds", StringUtilities.DblToString(effectiveSeconds, 1)); writer.WriteEndElement(); if (scanList.ProcessingIncomplete) { writer.WriteElementString("ProcessingComplete", "False"); } else { writer.WriteElementString("ProcessingComplete", "True"); } writer.WriteEndElement(); // Close out the <SICData> element writer.WriteEndDocument(); writer.Close(); } catch (Exception ex) { ReportError("Error finalizing the XML output file", ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } return(true); }
/// <summary> /// Validate that .MasterScanOrder() really is sorted by scan number /// </summary> /// <param name="scanList"></param> private void ValidateMasterScanOrderSorting(clsScanList scanList) { // Cannot use an IComparer because .MasterScanOrder points into other arrays var masterScanNumbers = new int[scanList.MasterScanOrderCount]; var masterScanOrderIndices = new int[scanList.MasterScanOrderCount]; for (var index = 0; index < scanList.MasterScanOrderCount; index++) { masterScanNumbers[index] = scanList.MasterScanNumList[index]; masterScanOrderIndices[index] = index; } // Sort masterScanNumbers ascending, sorting the scan order indices array in parallel Array.Sort(masterScanNumbers, masterScanOrderIndices); // Check whether we need to re-populate the lists var needToSort = false; for (var index = 1; index < scanList.MasterScanOrderCount; index++) { if (masterScanOrderIndices[index] < masterScanOrderIndices[index - 1]) { needToSort = true; break; } } if (needToSort) { // Reorder .MasterScanOrder, .MasterScanNumList, and .MasterScanTimeList var udtMasterScanOrderListCopy = new clsScanList.udtScanOrderPointerType[scanList.MasterScanOrder.Count]; var masterScanTimeListCopy = new float[scanList.MasterScanOrder.Count]; Array.Copy(scanList.MasterScanOrder.ToArray(), udtMasterScanOrderListCopy, scanList.MasterScanOrderCount); Array.Copy(scanList.MasterScanTimeList.ToArray(), masterScanTimeListCopy, scanList.MasterScanOrderCount); for (var index = 0; index < scanList.MasterScanOrderCount; index++) { scanList.MasterScanOrder[index] = udtMasterScanOrderListCopy[masterScanOrderIndices[index]]; scanList.MasterScanNumList[index] = masterScanNumbers[index]; scanList.MasterScanTimeList[index] = masterScanTimeListCopy[masterScanOrderIndices[index]]; } } }
private bool ExtractScanInfoCheckRange( XRawFileIO xcaliburAccessor, ThermoRawFileReader.clsScanInfo thermoScanInfo, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, double percentComplete) { bool success; if (mScanTracking.CheckScanInRange(thermoScanInfo.ScanNumber, thermoScanInfo.RetentionTime, mOptions.SICOptions)) { success = ExtractScanInfoWork(xcaliburAccessor, scanList, spectraCache, dataOutputHandler, mOptions.SICOptions, thermoScanInfo); } else { mScansOutOfRange += 1; success = true; } UpdateProgress((short)Math.Round(percentComplete, 0)); UpdateCacheStats(spectraCache); if (mOptions.AbortProcessing) { scanList.ProcessingIncomplete = true; return(false); } if (DateTime.UtcNow.Subtract(mLastLogTime).TotalSeconds >= 10 || thermoScanInfo.ScanNumber % 500 == 0 && ( thermoScanInfo.ScanNumber >= mOptions.SICOptions.ScanRangeStart && thermoScanInfo.ScanNumber <= mOptions.SICOptions.ScanRangeEnd)) { ReportMessage("Reading scan: " + thermoScanInfo.ScanNumber.ToString()); Console.Write("."); mLastLogTime = DateTime.UtcNow; } return(success); }
private float ScanNumberToScanTime( clsScanList scanList, int scanNumber) { var surveyScanMatches = (from item in scanList.SurveyScans where item.ScanNumber == scanNumber select item).ToList(); if (surveyScanMatches.Count > 0) { return(surveyScanMatches.First().ScanTime); } var fragScanMatches = (from item in scanList.FragScans where item.ScanNumber == scanNumber select item).ToList(); if (fragScanMatches.Count > 0) { return(fragScanMatches.First().ScanTime); } return(0); }
protected bool FinalizeScanList(clsScanList scanList, FileSystemInfo dataFile) { if (scanList.MasterScanOrderCount <= 0) { // No scans found if (mScansOutOfRange > 0) { ReportWarning("None of the spectra in the input file was within the specified scan number and/or scan time range: " + dataFile.FullName); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.NoParentIonsFoundInInputFile); } else { ReportError("No scans found in the input file: " + dataFile.FullName); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); } return(false); } if (mPrecursorNotFoundCount > PRECURSOR_NOT_FOUND_WARNINGS_TO_SHOW) { var precursorMissingPct = 0.0; if (scanList.FragScans.Count > 0) { precursorMissingPct = mPrecursorNotFoundCount / (double)scanList.FragScans.Count * 100; } OnWarningEvent( string.Format("Could not find the precursor ion for {0:F1}% of the MS2 spectra ({1} / {2} scans). " + "These scans will have an Interference Score of 1 (to avoid accidentally filtering out low intensity results).", precursorMissingPct, mPrecursorNotFoundCount, scanList.FragScans.Count)); } // Record the current memory usage OnUpdateMemoryUsage(); return(true); }
private void WriteSICStatsFlatFileEntry( TextWriter sicStatsWriter, char delimiter, clsSICOptions sicOptions, clsScanList scanList, clsParentIonInfo parentIon, int parentIonIndex, int surveyScanNumber, float surveyScanTime, int fragScanIndex, bool includeScanTimesInSICStatsFile) { var dataValues = new List <string>(40); float fragScanTime = 0; float optimalPeakApexScanTime = 0; dataValues.Add(sicOptions.DatasetID.ToString()); // Dataset ID dataValues.Add(parentIonIndex.ToString()); // Parent Ion Index dataValues.Add(StringUtilities.DblToString(parentIon.MZ, 4)); // MZ dataValues.Add(surveyScanNumber.ToString()); // Survey scan number double interferenceScore; if (fragScanIndex < scanList.FragScans.Count) { var fragScanNumber = scanList.FragScans[parentIon.FragScanIndices[fragScanIndex]].ScanNumber; dataValues.Add(fragScanNumber.ToString()); // Fragmentation scan number interferenceScore = scanList.FragScans[parentIon.FragScanIndices[fragScanIndex]].FragScanInfo.InterferenceScore; } else { dataValues.Add("0"); // Fragmentation scan does not exist interferenceScore = 0; } dataValues.Add(parentIon.OptimalPeakApexScanNumber.ToString()); // Optimal peak apex scan number if (includeScanTimesInSICStatsFile) { if (fragScanIndex < scanList.FragScans.Count) { fragScanTime = scanList.FragScans[parentIon.FragScanIndices[fragScanIndex]].ScanTime; } else { fragScanTime = 0; // Fragmentation scan does not exist } optimalPeakApexScanTime = ScanNumberToScanTime(scanList, parentIon.OptimalPeakApexScanNumber); } dataValues.Add(parentIon.PeakApexOverrideParentIonIndex.ToString()); // Parent Ion Index that supplied the optimal peak apex scan number if (parentIon.CustomSICPeak) { dataValues.Add("1"); // Custom SIC peak, record 1 } else { dataValues.Add("0"); // Not a Custom SIC peak, record 0 } var currentSIC = parentIon.SICStats; if (currentSIC.ScanTypeForPeakIndices == clsScanList.eScanTypeConstants.FragScan) { dataValues.Add(scanList.FragScans[currentSIC.PeakScanIndexStart].ScanNumber.ToString()); // Peak Scan Start dataValues.Add(scanList.FragScans[currentSIC.PeakScanIndexEnd].ScanNumber.ToString()); // Peak Scan End dataValues.Add(scanList.FragScans[currentSIC.PeakScanIndexMax].ScanNumber.ToString()); // Peak Scan Max Intensity } else { dataValues.Add(scanList.SurveyScans[currentSIC.PeakScanIndexStart].ScanNumber.ToString()); // Peak Scan Start dataValues.Add(scanList.SurveyScans[currentSIC.PeakScanIndexEnd].ScanNumber.ToString()); // Peak Scan End dataValues.Add(scanList.SurveyScans[currentSIC.PeakScanIndexMax].ScanNumber.ToString()); // Peak Scan Max Intensity } var currentPeak = currentSIC.Peak; dataValues.Add(StringUtilities.ValueToString(currentPeak.MaxIntensityValue, 5)); // Peak Intensity dataValues.Add(StringUtilities.ValueToString(currentPeak.SignalToNoiseRatio, 4)); // Peak signal to noise ratio dataValues.Add(currentPeak.FWHMScanWidth.ToString()); // Full width at half max (in scans) dataValues.Add(StringUtilities.ValueToString(currentPeak.Area, 5)); // Peak area dataValues.Add(StringUtilities.ValueToString(currentPeak.ParentIonIntensity, 5)); // Intensity of the parent ion (just before the fragmentation scan) dataValues.Add(StringUtilities.ValueToString(currentPeak.BaselineNoiseStats.NoiseLevel, 5)); dataValues.Add(StringUtilities.ValueToString(currentPeak.BaselineNoiseStats.NoiseStDev, 3)); dataValues.Add(currentPeak.BaselineNoiseStats.PointsUsed.ToString()); var statMoments = currentPeak.StatisticalMoments; dataValues.Add(StringUtilities.ValueToString(statMoments.Area, 5)); dataValues.Add(statMoments.CenterOfMassScan.ToString()); dataValues.Add(StringUtilities.ValueToString(statMoments.StDev, 3)); dataValues.Add(StringUtilities.ValueToString(statMoments.Skew, 4)); dataValues.Add(StringUtilities.ValueToString(statMoments.KSStat, 4)); dataValues.Add(statMoments.DataCountUsed.ToString()); dataValues.Add(StringUtilities.ValueToString(interferenceScore, 4)); // Interference Score if (includeScanTimesInSICStatsFile) { dataValues.Add(StringUtilities.DblToString(surveyScanTime, 5)); // SurveyScanTime dataValues.Add(StringUtilities.DblToString(fragScanTime, 5)); // FragScanTime dataValues.Add(StringUtilities.DblToString(optimalPeakApexScanTime, 5)); // OptimalPeakApexScanTime } sicStatsWriter.WriteLine(string.Join(delimiter.ToString(), dataValues)); }
public bool ExtractScanInfoFromMGFandCDF( string filePath, clsScanList scanList, clsSpectraCache spectraCache, DataOutput.clsDataOutput dataOutputHandler, bool keepRawSpectra, bool keepMSMSSpectra) { // Returns True if Success, False if failure // Note: This function assumes filePath exists // // This function can be used to read a pair of MGF and NetCDF files that contain MS/MS and MS-only parent ion scans, respectively // Typically, this will apply to LC-MS/MS analyses acquired using an Agilent mass spectrometer running DataAnalysis software // filePath can contain the path to the MGF or to the CDF file; the extension will be removed in order to determine the base file name, // then the two files will be looked for separately var scanTime = 0.0; var cdfReader = new NetCDFReader.clsMSNetCdf(); var mgfReader = new MSDataFileReader.clsMGFFileReader(); try { Console.Write("Reading CDF/MGF data files "); ReportMessage("Reading CDF/MGF data files"); UpdateProgress(0, "Opening data file: " + Environment.NewLine + Path.GetFileName(filePath)); // Obtain the full path to the file var mgfFileInfo = new FileInfo(filePath); var mgfInputFilePathFull = mgfFileInfo.FullName; // Make sure the extension for mgfInputFilePathFull is .MGF mgfInputFilePathFull = Path.ChangeExtension(mgfInputFilePathFull, AGILENT_MSMS_FILE_EXTENSION); var cdfInputFilePathFull = Path.ChangeExtension(mgfInputFilePathFull, AGILENT_MS_FILE_EXTENSION); var datasetID = mOptions.SICOptions.DatasetID; var sicOptions = mOptions.SICOptions; var success = UpdateDatasetFileStats(mgfFileInfo, datasetID); mDatasetFileInfo.ScanCount = 0; // Open a handle to each data file if (!cdfReader.OpenMSCdfFile(cdfInputFilePathFull)) { ReportError("Error opening input data file: " + cdfInputFilePathFull); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } if (!mgfReader.OpenFile(mgfInputFilePathFull)) { ReportError("Error opening input data file: " + mgfInputFilePathFull); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } var msScanCount = cdfReader.GetScanCount(); mDatasetFileInfo.ScanCount = msScanCount; if (msScanCount <= 0) { // No scans found ReportError("No scans found in the input file: " + cdfInputFilePathFull); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } // Reserve memory for all of the Survey Scan data scanList.Initialize(); var scanCount = mOptions.SICOptions.ScanRangeCount; if (scanCount <= 0) { scanCount = msScanCount * 3; } scanList.ReserveListCapacity(scanCount, Math.Min(msScanCount, scanCount / 3)); // Assume there are 2 frag scans for every survey scan mScanTracking.ReserveListCapacity(scanCount); spectraCache.SpectrumCount = Math.Max(spectraCache.SpectrumCount, scanCount); UpdateProgress("Reading CDF/MGF data (" + msScanCount.ToString() + " scans)" + Environment.NewLine + Path.GetFileName(filePath)); ReportMessage("Reading CDF/MGF data; Total MS scan count: " + msScanCount.ToString()); // Read all of the Survey scans from the CDF file // CDF files created by the Agilent XCT list the first scan number as 0; use scanNumberCorrection to correct for this var scanNumberCorrection = 0; for (var msScanIndex = 0; msScanIndex < msScanCount; msScanIndex++) { success = cdfReader.GetScanInfo(msScanIndex, out var scanNumber, out var scanTotalIntensity, out scanTime, out _, out _); if (msScanIndex == 0 && scanNumber == 0) { scanNumberCorrection = 1; } if (!success) { // Error reading CDF file ReportError("Error obtaining data from CDF file: " + cdfInputFilePathFull); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileDataReadError); return(false); } if (scanNumberCorrection > 0) { scanNumber += scanNumberCorrection; } if (mScanTracking.CheckScanInRange(scanNumber, scanTime, sicOptions)) { var newSurveyScan = new clsScanInfo { ScanNumber = scanNumber, TotalIonIntensity = (float)scanTotalIntensity, // Copy the Total Scan Intensity to .TotalIonIntensity ScanHeaderText = string.Empty, ScanTypeName = "MS", }; if (mOptions.CDFTimeInSeconds) { newSurveyScan.ScanTime = (float)(scanTime / 60); } else { newSurveyScan.ScanTime = (float)scanTime; } // Survey scans typically lead to multiple parent ions; we do not record them here newSurveyScan.FragScanInfo.ParentIonInfoIndex = -1; scanList.SurveyScans.Add(newSurveyScan); success = cdfReader.GetMassSpectrum(msScanIndex, out var mzData, out var intensityData, out var intIonCount, out _); if (success && intIonCount > 0) { var msSpectrum = new clsMSSpectrum(scanNumber, mzData, intensityData, intIonCount); double msDataResolution; newSurveyScan.IonCount = msSpectrum.IonCount; newSurveyScan.IonCountRaw = newSurveyScan.IonCount; // Find the base peak ion mass and intensity newSurveyScan.BasePeakIonMZ = FindBasePeakIon(msSpectrum.IonsMZ, msSpectrum.IonsIntensity, out var basePeakIonIntensity, out var mzMin, out var mzMax); newSurveyScan.BasePeakIonIntensity = basePeakIonIntensity; // Determine the minimum positive intensity in this scan newSurveyScan.MinimumPositiveIntensity = mPeakFinder.FindMinimumPositiveValue(msSpectrum.IonsIntensity, 0); if (sicOptions.SICToleranceIsPPM) { // Define MSDataResolution based on the tolerance value that will be used at the lowest m/z in this spectrum, divided by COMPRESS_TOLERANCE_DIVISOR // However, if the lowest m/z value is < 100, then use 100 m/z if (mzMin < 100) { msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, 100) / sicOptions.CompressToleranceDivisorForPPM; } else { msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, mzMin) / sicOptions.CompressToleranceDivisorForPPM; } } else { msDataResolution = sicOptions.SICTolerance / sicOptions.CompressToleranceDivisorForDa; } mScanTracking.ProcessAndStoreSpectrum( newSurveyScan, this, spectraCache, msSpectrum, sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions, clsMASIC.DISCARD_LOW_INTENSITY_MS_DATA_ON_LOAD, sicOptions.CompressMSSpectraData, msDataResolution, keepRawSpectra); } else { newSurveyScan.IonCount = 0; newSurveyScan.IonCountRaw = 0; } // Note: Since we're reading all of the Survey Scan data, we cannot update .MasterScanOrder() at this time } // Note: We need to take msScanCount * 2 since we have to read two different files if (msScanCount > 1) { UpdateProgress((short)(msScanIndex / (double)(msScanCount * 2 - 1) * 100)); } else { UpdateProgress(0); } UpdateCacheStats(spectraCache); if (mOptions.AbortProcessing) { scanList.ProcessingIncomplete = true; break; } if (msScanIndex % 100 == 0) { ReportMessage("Reading MS scan index: " + msScanIndex.ToString()); Console.Write("."); } } // Record the current memory usage (before we close the .CDF file) OnUpdateMemoryUsage(); cdfReader.CloseMSCdfFile(); // We loaded all of the survey scan data above // We can now initialize .MasterScanOrder() var lastSurveyScanIndex = 0; scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, lastSurveyScanIndex); var surveyScansRecorded = new SortedSet <int>() { lastSurveyScanIndex }; // Reset scanNumberCorrection; we might also apply it to MS/MS data scanNumberCorrection = 0; // Now read the MS/MS data from the MGF file do { var fragScanFound = mgfReader.ReadNextSpectrum(out var spectrumInfo); if (!fragScanFound) { break; } mDatasetFileInfo.ScanCount += 1; while (spectrumInfo.ScanNumber < scanList.SurveyScans[lastSurveyScanIndex].ScanNumber) { // The scan number for the current MS/MS spectrum is less than the last survey scan index scan number // This can happen, due to oddities with combining scans when creating the .MGF file // Need to decrement lastSurveyScanIndex until we find the appropriate survey scan lastSurveyScanIndex -= 1; if (lastSurveyScanIndex == 0) { break; } } if (scanNumberCorrection == 0) { // See if udtSpectrumHeaderInfo.ScanNumberStart is equivalent to one of the survey scan numbers, yielding conflicting scan numbers // If it is, then there is an indexing error in the .MGF file; this error was present in .MGF files generated with // an older version of Agilent Chemstation. These files typically have lines like ###MSMS: #13-29 instead of ###MSMS: #13/29/ // If this indexing error is found, then we'll set scanNumberCorrection = 1 and apply it to all subsequent MS/MS scans; // we'll also need to correct prior MS/MS scans for (var surveyScanIndex = lastSurveyScanIndex; surveyScanIndex < scanList.SurveyScans.Count; surveyScanIndex++) { if (scanList.SurveyScans[surveyScanIndex].ScanNumber == spectrumInfo.ScanNumber) { // Conflicting scan numbers were found scanNumberCorrection = 1; // Need to update prior MS/MS scans foreach (var fragScan in scanList.FragScans) { fragScan.ScanNumber += scanNumberCorrection; var scanTimeInterpolated = InterpolateRTandFragScanNumber( scanList.SurveyScans, 0, fragScan.ScanNumber, out var fragScanIterationOut); fragScan.FragScanInfo.FragScanNumber = fragScanIterationOut; fragScan.ScanTime = scanTimeInterpolated; } break; } if (scanList.SurveyScans[surveyScanIndex].ScanNumber > spectrumInfo.ScanNumber) { break; } } } if (scanNumberCorrection > 0) { spectrumInfo.ScanNumber += scanNumberCorrection; spectrumInfo.ScanNumberEnd += scanNumberCorrection; } scanTime = InterpolateRTandFragScanNumber( scanList.SurveyScans, lastSurveyScanIndex, spectrumInfo.ScanNumber, out var fragScanIteration); // Make sure this fragmentation scan isn't present yet in scanList.FragScans // This can occur in Agilent .MGF files if the scan is listed both singly and grouped with other MS/MS scans var validFragScan = true; foreach (var fragScan in scanList.FragScans) { if (fragScan.ScanNumber == spectrumInfo.ScanNumber) { // Duplicate found validFragScan = false; break; } } if (!(validFragScan && mScanTracking.CheckScanInRange(spectrumInfo.ScanNumber, scanTime, sicOptions))) { continue; } // See if lastSurveyScanIndex needs to be updated // At the same time, populate .MasterScanOrder while (lastSurveyScanIndex < scanList.SurveyScans.Count - 1 && spectrumInfo.ScanNumber > scanList.SurveyScans[lastSurveyScanIndex + 1].ScanNumber) { lastSurveyScanIndex += 1; // Add the given SurveyScan to .MasterScanOrder, though only if it hasn't yet been added if (!surveyScansRecorded.Contains(lastSurveyScanIndex)) { surveyScansRecorded.Add(lastSurveyScanIndex); scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, lastSurveyScanIndex); } } scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.FragScan, scanList.FragScans.Count, spectrumInfo.ScanNumber, (float)scanTime); var newFragScan = new clsScanInfo { ScanNumber = spectrumInfo.ScanNumber, ScanTime = (float)scanTime, ScanHeaderText = string.Empty, ScanTypeName = "MSn", }; newFragScan.FragScanInfo.FragScanNumber = fragScanIteration; newFragScan.FragScanInfo.MSLevel = 2; newFragScan.MRMScanInfo.MRMMassCount = 0; scanList.FragScans.Add(newFragScan); var msSpectrum = new clsMSSpectrum(newFragScan.ScanNumber, spectrumInfo.MZList, spectrumInfo.IntensityList, spectrumInfo.DataCount); if (spectrumInfo.DataCount > 0) { newFragScan.IonCount = msSpectrum.IonCount; newFragScan.IonCountRaw = newFragScan.IonCount; // Find the base peak ion mass and intensity newFragScan.BasePeakIonMZ = FindBasePeakIon(msSpectrum.IonsMZ, msSpectrum.IonsIntensity, out var basePeakIonIntensity, out _, out _); newFragScan.BasePeakIonIntensity = basePeakIonIntensity; // Compute the total scan intensity newFragScan.TotalIonIntensity = 0; for (var ionIndex = 0; ionIndex < newFragScan.IonCount; ionIndex++) { newFragScan.TotalIonIntensity += msSpectrum.IonsIntensity[ionIndex]; } // Determine the minimum positive intensity in this scan newFragScan.MinimumPositiveIntensity = mPeakFinder.FindMinimumPositiveValue(msSpectrum.IonsIntensity, 0); var msDataResolution = mOptions.BinningOptions.BinSize / sicOptions.CompressToleranceDivisorForDa; var keepRawSpectrum = keepRawSpectra && keepMSMSSpectra; mScanTracking.ProcessAndStoreSpectrum( newFragScan, this, spectraCache, msSpectrum, sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions, clsMASIC.DISCARD_LOW_INTENSITY_MSMS_DATA_ON_LOAD, sicOptions.CompressMSMSSpectraData, msDataResolution, keepRawSpectrum); } else { newFragScan.IonCount = 0; newFragScan.IonCountRaw = 0; newFragScan.TotalIonIntensity = 0; } mParentIonProcessor.AddUpdateParentIons(scanList, lastSurveyScanIndex, spectrumInfo.ParentIonMZ, scanList.FragScans.Count - 1, spectraCache, sicOptions); // Note: We need to take msScanCount * 2, in addition to adding msScanCount to lastSurveyScanIndex, since we have to read two different files if (msScanCount > 1) { UpdateProgress((short)((lastSurveyScanIndex + msScanCount) / (double)(msScanCount * 2 - 1) * 100)); } else { UpdateProgress(0); } UpdateCacheStats(spectraCache); if (mOptions.AbortProcessing) { scanList.ProcessingIncomplete = true; break; } if (scanList.FragScans.Count % 100 == 0) { ReportMessage("Reading MSMS scan index: " + scanList.FragScans.Count); Console.Write("."); } }while (true); // Record the current memory usage (before we close the .MGF file) OnUpdateMemoryUsage(); mgfReader.CloseFile(); // Check for any other survey scans that need to be added to MasterScanOrder // See if lastSurveyScanIndex needs to be updated // At the same time, populate .MasterScanOrder while (lastSurveyScanIndex < scanList.SurveyScans.Count - 1) { lastSurveyScanIndex += 1; // Note that scanTime is the scan time of the most recent survey scan processed in the above Do loop, so it's not accurate if (mScanTracking.CheckScanInRange(scanList.SurveyScans[lastSurveyScanIndex].ScanNumber, scanTime, sicOptions)) { // Add the given SurveyScan to .MasterScanOrder, though only if it hasn't yet been added if (!surveyScansRecorded.Contains(lastSurveyScanIndex)) { surveyScansRecorded.Add(lastSurveyScanIndex); scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, lastSurveyScanIndex); } } } // Make sure that MasterScanOrder really is sorted by scan number ValidateMasterScanOrderSorting(scanList); // Now that all of the data has been read, write out to the scan stats file, in order of scan number for (var scanIndex = 0; scanIndex < scanList.MasterScanOrderCount; scanIndex++) { var eScanType = scanList.MasterScanOrder[scanIndex].ScanType; clsScanInfo currentScan; if (eScanType == clsScanList.eScanTypeConstants.SurveyScan) { // Survey scan currentScan = scanList.SurveyScans[scanList.MasterScanOrder[scanIndex].ScanIndexPointer]; } else { // Frag Scan currentScan = scanList.FragScans[scanList.MasterScanOrder[scanIndex].ScanIndexPointer]; } SaveScanStatEntry(dataOutputHandler.OutputFileHandles.ScanStats, eScanType, currentScan, sicOptions.DatasetID); } Console.WriteLine(); scanList.SetListCapacityToCount(); mScanTracking.SetListCapacityToCount(); return(success); } catch (Exception ex) { ReportError("Error in ExtractScanInfoFromMGFandCDF", ex, clsMASIC.eMasicErrorCodes.InputFileDataReadError); return(false); } }
public bool SaveExtendedScanStatsFiles( clsScanList scanList, string inputFileName, string outputDirectoryPath, bool includeHeaders) { // Writes out a flat file containing the extended scan stats var extendedNonConstantHeaderOutputFilePath = string.Empty; const char TAB_DELIMITER = '\t'; try { UpdateProgress(0, "Saving extended scan stats to flat file"); var extendedConstantHeaderOutputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.ScanStatsExtendedConstantFlatFile); extendedNonConstantHeaderOutputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.ScanStatsExtendedFlatFile); ReportMessage("Saving extended scan stats flat file to disk: " + Path.GetFileName(extendedNonConstantHeaderOutputFilePath)); if (mExtendedHeaderNameMap.Count == 0) { // No extended stats to write; exit the function UpdateProgress(100); return(true); } // Lookup extended stats values that are constants for all scans // The following will also remove the constant header values from mExtendedHeaderNameMap var constantExtendedHeaderValues = ExtractConstantExtendedHeaderValues(out var nonConstantHeaderIDs, scanList.SurveyScans, scanList.FragScans, TAB_DELIMITER); if (constantExtendedHeaderValues == null) { constantExtendedHeaderValues = string.Empty; } // Write the constant extended stats values to a text file using (var writer = new StreamWriter(extendedConstantHeaderOutputFilePath, false)) { writer.WriteLine(constantExtendedHeaderValues); } // Now open another output file for the non-constant extended stats using (var writer = new StreamWriter(extendedNonConstantHeaderOutputFilePath, false)) { if (includeHeaders) { var headerNames = ConstructExtendedStatsHeaders(); writer.WriteLine(string.Join(TAB_DELIMITER.ToString(), headerNames)); } for (var scanIndex = 0; scanIndex < scanList.MasterScanOrderCount; scanIndex++) { var currentScan = GetScanByMasterScanIndex(scanList, scanIndex); var dataColumns = ConcatenateExtendedStats(nonConstantHeaderIDs, mOptions.SICOptions.DatasetID, currentScan.ScanNumber, currentScan.ExtendedHeaderInfo); writer.WriteLine(string.Join(TAB_DELIMITER.ToString(), dataColumns)); if (scanIndex % 100 == 0) { UpdateProgress((short)(scanIndex / (double)(scanList.MasterScanOrderCount - 1) * 100)); } } } } catch (Exception ex) { ReportError("Error writing the Extended Scan Stats to: " + extendedNonConstantHeaderOutputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } UpdateProgress(100); return(true); }
/// <summary> /// Read scan data and ions from a Thermo .raw file /// </summary> /// <param name="filePath"></param> /// <param name="scanList"></param> /// <param name="spectraCache"></param> /// <param name="dataOutputHandler"></param> /// <param name="keepRawSpectra"></param> /// <param name="keepMSMSSpectra"></param> /// <returns>True if Success, False if failure</returns> /// <remarks>Assumes filePath exists</remarks> public bool ExtractScanInfoFromXcaliburDataFile( string filePath, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, bool keepRawSpectra, bool keepMSMSSpectra) { // Use XrawFileIO to read the .Raw files (it uses ThermoFisher.CommonCore) var readerOptions = new ThermoReaderOptions { LoadMSMethodInfo = mOptions.WriteMSMethodFile, LoadMSTuneInfo = mOptions.WriteMSTuneFile }; var xcaliburAccessor = new XRawFileIO(readerOptions) { ScanInfoCacheMaxSize = 0 // Don't cache scanInfo objects }; RegisterEvents(xcaliburAccessor); mBpiUpdateCount = 0; // Assume success for now var success = true; try { Console.Write("Reading Thermo .raw file "); ReportMessage("Reading Thermo .raw file"); UpdateProgress(0, "Opening data file:" + Environment.NewLine + Path.GetFileName(filePath)); // Obtain the full path to the file var rawFileInfo = new FileInfo(filePath); var inputFileFullPath = rawFileInfo.FullName; // Open a handle to the data file if (!xcaliburAccessor.OpenRawFile(inputFileFullPath)) { ReportError("Error opening input data file: " + inputFileFullPath + " (xcaliburAccessor.OpenRawFile returned False)"); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } var datasetID = mOptions.SICOptions.DatasetID; success = UpdateDatasetFileStats(rawFileInfo, datasetID, xcaliburAccessor); var metadataWriter = new clsThermoMetadataWriter(); RegisterEvents(metadataWriter); if (mOptions.WriteMSMethodFile) { metadataWriter.SaveMSMethodFile(xcaliburAccessor, dataOutputHandler); } if (mOptions.WriteMSTuneFile) { metadataWriter.SaveMSTuneFile(xcaliburAccessor, dataOutputHandler); } var scanCount = xcaliburAccessor.GetNumScans(); if (scanCount <= 0) { // No scans found ReportError("No scans found in the input file: " + filePath); SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError); return(false); } var scanStart = xcaliburAccessor.ScanStart; var scanEnd = xcaliburAccessor.ScanEnd; InitOptions(scanList, keepRawSpectra, keepMSMSSpectra); UpdateProgress(string.Format("Reading Xcalibur data ({0:N0} scans){1}", scanCount, Environment.NewLine + Path.GetFileName(filePath))); ReportMessage(string.Format("Reading Xcalibur data; Total scan count: {0:N0}", scanCount)); var scanCountToRead = scanEnd - scanStart + 1; var scansEst = mOptions.SICOptions.ScanRangeCount; if (scansEst <= 0) { scansEst = scanCountToRead; } scanList.ReserveListCapacity(scansEst); mScanTracking.ReserveListCapacity(scansEst); spectraCache.SpectrumCount = Math.Max(spectraCache.SpectrumCount, scansEst); for (var scanNumber = scanStart; scanNumber <= scanEnd; scanNumber++) { if (!mScanTracking.CheckScanInRange(scanNumber, mOptions.SICOptions)) { mScansOutOfRange += 1; continue; } success = xcaliburAccessor.GetScanInfo(scanNumber, out ThermoRawFileReader.clsScanInfo thermoScanInfo); if (!success) { // GetScanInfo returned false ReportWarning("xcaliburAccessor.GetScanInfo returned false for scan " + scanNumber.ToString() + "; aborting read"); break; } var percentComplete = scanList.MasterScanOrderCount / (double)(scanCountToRead) * 100; var extractSuccess = ExtractScanInfoCheckRange(xcaliburAccessor, thermoScanInfo, scanList, spectraCache, dataOutputHandler, percentComplete); if (!extractSuccess) { break; } } Console.WriteLine(); scanList.SetListCapacityToCount(); mScanTracking.SetListCapacityToCount(); // Shrink the memory usage of the scanList arrays success = FinalizeScanList(scanList, rawFileInfo); } catch (Exception ex) { Console.WriteLine(ex.StackTrace); ReportError("Error in ExtractScanInfoFromXcaliburDataFile", ex, clsMASIC.eMasicErrorCodes.InputFileDataReadError); } // Close the handle to the data file xcaliburAccessor.CloseRawFile(); return(success); }
public void TestScanConversions() { const double MZ_MINIMUM = 100; const float INTENSITY_MINIMUM = 10000; const float SCAN_TIME_SCALAR = 10; var scanList = new clsScanList(); var oRand = new Random(); var intLastSurveyScanIndexInMasterSeqOrder = -1; // Populate scanList with example scan data for (var scanNumber = 1; scanNumber <= 1750; scanNumber++) { if (scanNumber % 10 == 0) { // Add a survey scan // If this is a mzXML file that was processed with ReadW, .ScanHeaderText and .ScanTypeName will get updated by UpdateMSXMLScanType var newSurveyScan = new MASIC.clsScanInfo { ScanNumber = scanNumber, ScanTime = scanNumber / SCAN_TIME_SCALAR, ScanHeaderText = string.Empty, ScanTypeName = "MS", BasePeakIonMZ = MZ_MINIMUM + oRand.NextDouble() * 1000, BasePeakIonIntensity = INTENSITY_MINIMUM + (float)oRand.NextDouble() * 1000 }; // Survey scans typically lead to multiple parent ions; we do not record them here newSurveyScan.FragScanInfo.ParentIonInfoIndex = -1; newSurveyScan.TotalIonIntensity = newSurveyScan.BasePeakIonIntensity * (float)(0.25 + oRand.NextDouble() * 5); // Determine the minimum positive intensity in this scan newSurveyScan.MinimumPositiveIntensity = INTENSITY_MINIMUM; // If this is a mzXML file that was processed with ReadW, then these values will get updated by UpdateMSXMLScanType newSurveyScan.ZoomScan = false; newSurveyScan.SIMScan = false; newSurveyScan.MRMScanType = MRMScanTypeConstants.NotMRM; newSurveyScan.LowMass = MZ_MINIMUM; newSurveyScan.HighMass = Math.Max(newSurveyScan.BasePeakIonMZ * 1.1, MZ_MINIMUM * 10); newSurveyScan.IsFTMS = false; scanList.SurveyScans.Add(newSurveyScan); var intLastSurveyScanIndex = scanList.SurveyScans.Count - 1; scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, intLastSurveyScanIndex); intLastSurveyScanIndexInMasterSeqOrder = scanList.MasterScanOrderCount - 1; } else { // If this is a mzXML file that was processed with ReadW, .ScanHeaderText and .ScanTypeName will get updated by UpdateMSXMLScanType var newFragScan = new MASIC.clsScanInfo { ScanNumber = scanNumber, ScanTime = scanNumber / SCAN_TIME_SCALAR, ScanHeaderText = string.Empty, ScanTypeName = "MSn", BasePeakIonMZ = MZ_MINIMUM + oRand.NextDouble() * 1000, BasePeakIonIntensity = INTENSITY_MINIMUM + (float)oRand.NextDouble() * 1000 }; // 1 for the first MS/MS scan after the survey scan, 2 for the second one, etc. newFragScan.FragScanInfo.FragScanNumber = (scanList.MasterScanOrderCount - 1) - intLastSurveyScanIndexInMasterSeqOrder; newFragScan.FragScanInfo.MSLevel = 2; newFragScan.TotalIonIntensity = newFragScan.BasePeakIonIntensity * (float)(0.25 + oRand.NextDouble() * 2); // Determine the minimum positive intensity in this scan newFragScan.MinimumPositiveIntensity = INTENSITY_MINIMUM; // If this is a mzXML file that was processed with ReadW, then these values will get updated by UpdateMSXMLScanType newFragScan.ZoomScan = false; newFragScan.SIMScan = false; newFragScan.MRMScanType = MRMScanTypeConstants.NotMRM; newFragScan.MRMScanInfo.MRMMassCount = 0; newFragScan.LowMass = MZ_MINIMUM; newFragScan.HighMass = Math.Max(newFragScan.BasePeakIonMZ * 1.1, MZ_MINIMUM * 10); newFragScan.IsFTMS = false; scanList.FragScans.Add(newFragScan); scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.FragScan, scanList.FragScans.Count - 1); } } var scanNumScanConverter = new clsScanNumScanTimeConversion(); RegisterEvents(scanNumScanConverter); // Convert absolute values // Scan 500, relative scan 0.5, and the scan at 30 minutes TestScanConversionToAbsolute( scanList, scanNumScanConverter, new KeyValuePair <int, int>(500, 500), new KeyValuePair <float, float>(0.5F, 876), new KeyValuePair <float, float>(30, 300)); TestScanConversionToTime( scanList, scanNumScanConverter, new KeyValuePair <int, int>(500, 50), new KeyValuePair <float, float>(0.5F, 87.55F), new KeyValuePair <float, float>(30, 30)); // Convert ranges // 50 scans wide, 10% of the run, and 5 minutes TestScanConversionToAbsolute( scanList, scanNumScanConverter, new KeyValuePair <int, int>(50, 50), new KeyValuePair <float, float>(0.1F, 176), new KeyValuePair <float, float>(5, 50)); TestScanConversionToTime( scanList, scanNumScanConverter, new KeyValuePair <int, int>(50, 5), new KeyValuePair <float, float>(0.1F, 17.59F), new KeyValuePair <float, float>(5, 5)); }
public bool ExportRawDataToDisk( clsScanList scanList, clsSpectraCache spectraCache, string inputFileName, string outputDirectoryPath) { var outputFilePath = "??"; try { StreamWriter dataWriter; StreamWriter scanInfoWriter; switch (mOptions.RawDataExportOptions.FileFormat) { case clsRawDataExportOptions.eExportRawDataFileFormatConstants.PEKFile: outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.PEKFile); dataWriter = new StreamWriter(outputFilePath); scanInfoWriter = null; break; case clsRawDataExportOptions.eExportRawDataFileFormatConstants.CSVFile: outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsIsosFile); var outputFilePath2 = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.DeconToolsScansFile); dataWriter = new StreamWriter(outputFilePath); scanInfoWriter = new StreamWriter(outputFilePath2); // Write the file headers mBPIWriter.WriteDecon2LSIsosFileHeaders(dataWriter); mBPIWriter.WriteDecon2LSScanFileHeaders(scanInfoWriter); break; default: // Unknown format ReportError("Unknown raw data file format: " + mOptions.RawDataExportOptions.FileFormat.ToString()); return(false); } var spectrumExportCount = 0; if (!mOptions.RawDataExportOptions.IncludeMSMS && mOptions.RawDataExportOptions.RenumberScans) { mOptions.RawDataExportOptions.RenumberScans = true; } else { mOptions.RawDataExportOptions.RenumberScans = false; } UpdateProgress(0, "Exporting raw data"); for (var masterOrderIndex = 0; masterOrderIndex < scanList.MasterScanOrderCount; masterOrderIndex++) { var scanPointer = scanList.MasterScanOrder[masterOrderIndex].ScanIndexPointer; if (scanList.MasterScanOrder[masterOrderIndex].ScanType == clsScanList.eScanTypeConstants.SurveyScan) { SaveRawDataToDiskWork(dataWriter, scanInfoWriter, scanList.SurveyScans[scanPointer], spectraCache, inputFileName, false, ref spectrumExportCount); } else if (mOptions.RawDataExportOptions.IncludeMSMS || scanList.FragScans[scanPointer].MRMScanType != ThermoRawFileReader.MRMScanTypeConstants.NotMRM) { // Either we're writing out MS/MS data or this is an MRM scan SaveRawDataToDiskWork(dataWriter, scanInfoWriter, scanList.FragScans[scanPointer], spectraCache, inputFileName, true, ref spectrumExportCount); } if (scanList.MasterScanOrderCount > 1) { UpdateProgress((short)(masterOrderIndex / (double)(scanList.MasterScanOrderCount - 1) * 100)); } else { UpdateProgress(0); } UpdateCacheStats(spectraCache); if (mOptions.AbortProcessing) { break; } } dataWriter.Close(); scanInfoWriter?.Close(); return(true); } catch (Exception ex) { ReportError("Error writing the raw spectra data to: " + outputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } }
/// <summary> /// Update inputFileName with optimal peak apex values /// </summary> /// <param name="scanList"></param> /// <param name="inputFileName"></param> /// <param name="outputDirectoryPath"></param> /// <returns></returns> public bool XmlOutputFileUpdateEntries( clsScanList scanList, string inputFileName, string outputDirectoryPath) { // ReSharper disable once StringLiteralTypo const string PARENT_ION_TAG_START_LOWER = "<parention"; // Note: this needs to be lowercase const string INDEX_ATTRIBUTE_LOWER = "index="; // Note: this needs to be lowercase const string OPTIMAL_PEAK_APEX_TAG_NAME = "OptimalPeakApexScanNumber"; const string PEAK_APEX_OVERRIDE_PARENT_ION_TAG_NAME = "PeakApexOverrideParentIonIndex"; var xmlReadFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.XMLFile); var xmlOutputFilePath = Path.Combine(outputDirectoryPath, "__temp__MASICOutputFile.xml"); try { // Wait 2 seconds before reopening the file, to make sure the handle is closed System.Threading.Thread.Sleep(2000); if (!File.Exists(xmlReadFilePath)) { // XML file not found, exit the function return(true); } using (var reader = new StreamReader(xmlReadFilePath)) using (var writer = new StreamWriter(xmlOutputFilePath, false)) { UpdateProgress(0, "Updating XML file with optimal peak apex values"); var parentIonIndex = -1; var parentIonsProcessed = 0; while (!reader.EndOfStream) { var dataLine = reader.ReadLine(); if (dataLine == null) { continue; } var dataLineLCase = dataLine.Trim().ToLower(); if (dataLineLCase.StartsWith(PARENT_ION_TAG_START_LOWER)) { var charIndex = dataLineLCase.IndexOf(INDEX_ATTRIBUTE_LOWER, StringComparison.CurrentCultureIgnoreCase); if (charIndex > 0) { var work = dataLineLCase.Substring(charIndex + INDEX_ATTRIBUTE_LOWER.Length + 1); charIndex = work.IndexOf('"'); if (charIndex > 0) { work = work.Substring(0, charIndex); if (clsUtilities.IsNumber(work)) { parentIonIndex = int.Parse(work); parentIonsProcessed += 1; // Update progress if (scanList.ParentIons.Count > 1) { if (parentIonsProcessed % 100 == 0) { UpdateProgress((short)(parentIonsProcessed / (double)(scanList.ParentIons.Count - 1) * 100)); } } else { UpdateProgress(0); } if (mOptions.AbortProcessing) { scanList.ProcessingIncomplete = true; break; } } } } writer.WriteLine(dataLine); } else if (dataLineLCase.StartsWith("<" + OPTIMAL_PEAK_APEX_TAG_NAME.ToLower()) && parentIonIndex >= 0) { if (parentIonIndex < scanList.ParentIons.Count) { XmlOutputFileReplaceSetting(writer, dataLine, OPTIMAL_PEAK_APEX_TAG_NAME, scanList.ParentIons[parentIonIndex].OptimalPeakApexScanNumber); } } else if (dataLineLCase.StartsWith("<" + PEAK_APEX_OVERRIDE_PARENT_ION_TAG_NAME.ToLower()) && parentIonIndex >= 0) { if (parentIonIndex < scanList.ParentIons.Count) { XmlOutputFileReplaceSetting(writer, dataLine, PEAK_APEX_OVERRIDE_PARENT_ION_TAG_NAME, scanList.ParentIons[parentIonIndex].PeakApexOverrideParentIonIndex); } } else { writer.WriteLine(dataLine); } } } try { // Wait 2 seconds, then delete the original file and rename the temp one to the original one System.Threading.Thread.Sleep(2000); if (File.Exists(xmlOutputFilePath)) { if (File.Exists(xmlReadFilePath)) { File.Delete(xmlReadFilePath); System.Threading.Thread.Sleep(500); } File.Move(xmlOutputFilePath, xmlReadFilePath); } } catch (Exception ex) { ReportError("Error renaming XML output file from temp name to: " + xmlReadFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } UpdateProgress(100); #if GUI System.Windows.Forms.Application.DoEvents(); #endif } catch (Exception ex) { ReportError("Error updating the XML output file: " + xmlReadFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } return(true); }
public bool SaveDataToXML( clsScanList scanList, int parentIonIndex, clsSICDetails sicDetails, MASICPeakFinder.clsSmoothedYDataSubset smoothedYDataSubset, clsDataOutput dataOutputHandler) { var lastGoodLoc = "Start"; try { // Populate SICDataScanIntervals with the scan intervals between each of the data points in sicDetails.SICScanNumbers // The first scan number is given by SICScanIndices(0) byte[] SICDataScanIntervals; if (sicDetails.SICDataCount == 0) { SICDataScanIntervals = new byte[1]; } else { SICDataScanIntervals = new byte[sicDetails.SICDataCount]; var sicScanNumbers = sicDetails.SICScanNumbers; for (var scanIndex = 1; scanIndex < sicDetails.SICDataCount; scanIndex++) { var scanDelta = sicScanNumbers[scanIndex] - sicScanNumbers[scanIndex - 1]; // When storing in SICDataScanIntervals, make sure the Scan Interval is, at most, 255; it will typically be 1 or 4 // However, for MRM data, field size can be much larger SICDataScanIntervals[scanIndex] = (byte)Math.Min(byte.MaxValue, scanDelta); } } var writer = dataOutputHandler.OutputFileHandles.XMLFileForSICs; if (writer == null) { return(false); } // Initialize the StringBuilder objects var sbIntensityDataList = new System.Text.StringBuilder(); var sbMassDataList = new System.Text.StringBuilder(); var sbPeakYDataSmoothed = new System.Text.StringBuilder(); var sicScanIndices = sicDetails.SICScanIndices; // Write the SIC's and computed peak stats and areas to the XML file for the given parent ion for (var fragScanIndex = 0; fragScanIndex < scanList.ParentIons[parentIonIndex].FragScanIndices.Count; fragScanIndex++) { lastGoodLoc = "fragScanIndex=" + fragScanIndex; writer.WriteStartElement("ParentIon"); writer.WriteAttributeString("Index", parentIonIndex.ToString()); // Parent ion Index writer.WriteAttributeString("FragScanIndex", fragScanIndex.ToString()); // Frag Scan Index lastGoodLoc = "currentParentIon = scanList.ParentIons(parentIonIndex)"; var currentParentIon = scanList.ParentIons[parentIonIndex]; writer.WriteElementString("MZ", StringUtilities.DblToString(currentParentIon.MZ, 4)); if (currentParentIon.SurveyScanIndex >= 0 && currentParentIon.SurveyScanIndex < scanList.SurveyScans.Count) { writer.WriteElementString("SurveyScanNumber", scanList.SurveyScans[currentParentIon.SurveyScanIndex].ScanNumber.ToString()); } else { writer.WriteElementString("SurveyScanNumber", "-1"); } lastGoodLoc = "Write FragScanNumber"; double interferenceScore; if (fragScanIndex < scanList.FragScans.Count) { var currentFragScan = scanList.FragScans[currentParentIon.FragScanIndices[fragScanIndex]]; writer.WriteElementString("FragScanNumber", currentFragScan.ScanNumber.ToString()); writer.WriteElementString("FragScanTime", currentFragScan.ScanTime.ToString(CultureInfo.InvariantCulture)); interferenceScore = currentFragScan.FragScanInfo.InterferenceScore; } else { // Fragmentation scan does not exist writer.WriteElementString("FragScanNumber", "0"); writer.WriteElementString("FragScanTime", "0"); interferenceScore = 0; } writer.WriteElementString("OptimalPeakApexScanNumber", currentParentIon.OptimalPeakApexScanNumber.ToString()); writer.WriteElementString("PeakApexOverrideParentIonIndex", currentParentIon.PeakApexOverrideParentIonIndex.ToString()); writer.WriteElementString("CustomSICPeak", currentParentIon.CustomSICPeak.ToString()); if (currentParentIon.CustomSICPeak) { writer.WriteElementString("CustomSICPeakComment", currentParentIon.CustomSICPeakComment); writer.WriteElementString("CustomSICPeakMZToleranceDa", currentParentIon.CustomSICPeakMZToleranceDa.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("CustomSICPeakScanTolerance", currentParentIon.CustomSICPeakScanOrAcqTimeTolerance.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("CustomSICPeakScanToleranceType", mOptions.CustomSICList.ScanToleranceType.ToString()); } lastGoodLoc = "sicStatsPeak = currentParentIon.SICStats.Peak"; var sicStatsPeak = currentParentIon.SICStats.Peak; if (sicDetails.SICScanType == clsScanList.eScanTypeConstants.FragScan) { writer.WriteElementString("SICScanType", "FragScan"); writer.WriteElementString("PeakScanStart", scanList.FragScans[sicScanIndices[sicStatsPeak.IndexBaseLeft]].ScanNumber.ToString()); writer.WriteElementString("PeakScanEnd", scanList.FragScans[sicScanIndices[sicStatsPeak.IndexBaseRight]].ScanNumber.ToString()); writer.WriteElementString("PeakScanMaxIntensity", scanList.FragScans[sicScanIndices[sicStatsPeak.IndexMax]].ScanNumber.ToString()); } else { writer.WriteElementString("SICScanType", "SurveyScan"); writer.WriteElementString("PeakScanStart", scanList.SurveyScans[sicScanIndices[sicStatsPeak.IndexBaseLeft]].ScanNumber.ToString()); writer.WriteElementString("PeakScanEnd", scanList.SurveyScans[sicScanIndices[sicStatsPeak.IndexBaseRight]].ScanNumber.ToString()); writer.WriteElementString("PeakScanMaxIntensity", scanList.SurveyScans[sicScanIndices[sicStatsPeak.IndexMax]].ScanNumber.ToString()); } writer.WriteElementString("PeakIntensity", StringUtilities.ValueToString(sicStatsPeak.MaxIntensityValue, 5)); writer.WriteElementString("PeakSignalToNoiseRatio", StringUtilities.ValueToString(sicStatsPeak.SignalToNoiseRatio, 4)); writer.WriteElementString("FWHMInScans", sicStatsPeak.FWHMScanWidth.ToString()); writer.WriteElementString("PeakArea", StringUtilities.ValueToString(sicStatsPeak.Area, 5)); writer.WriteElementString("ShoulderCount", sicStatsPeak.ShoulderCount.ToString()); writer.WriteElementString("ParentIonIntensity", StringUtilities.ValueToString(sicStatsPeak.ParentIonIntensity, 5)); var noiseStats = sicStatsPeak.BaselineNoiseStats; writer.WriteElementString("PeakBaselineNoiseLevel", StringUtilities.ValueToString(noiseStats.NoiseLevel, 5)); writer.WriteElementString("PeakBaselineNoiseStDev", StringUtilities.ValueToString(noiseStats.NoiseStDev, 3)); writer.WriteElementString("PeakBaselinePointsUsed", noiseStats.PointsUsed.ToString()); writer.WriteElementString("NoiseThresholdModeUsed", ((int)noiseStats.NoiseThresholdModeUsed).ToString()); var statMoments = sicStatsPeak.StatisticalMoments; writer.WriteElementString("StatMomentsArea", StringUtilities.ValueToString(statMoments.Area, 5)); writer.WriteElementString("CenterOfMassScan", statMoments.CenterOfMassScan.ToString()); writer.WriteElementString("PeakStDev", StringUtilities.ValueToString(statMoments.StDev, 3)); writer.WriteElementString("PeakSkew", StringUtilities.ValueToString(statMoments.Skew, 4)); writer.WriteElementString("PeakKSStat", StringUtilities.ValueToString(statMoments.KSStat, 4)); writer.WriteElementString("StatMomentsDataCountUsed", statMoments.DataCountUsed.ToString()); writer.WriteElementString("InterferenceScore", StringUtilities.ValueToString(interferenceScore, 4)); if (sicDetails.SICScanType == clsScanList.eScanTypeConstants.FragScan) { writer.WriteElementString("SICScanStart", scanList.FragScans[sicScanIndices[0]].ScanNumber.ToString()); } else { writer.WriteElementString("SICScanStart", scanList.SurveyScans[sicScanIndices[0]].ScanNumber.ToString()); } if (mOptions.UseBase64DataEncoding) { // Save scan interval list as base-64 encoded strings lastGoodLoc = "Call SaveDataToXMLEncodeArray with SICScanIntervals"; SaveDataToXMLEncodeArray(writer, "SICScanIntervals", SICDataScanIntervals); } else { // Save scan interval list as long list of numbers // There are no tab delimiters, since we require that all // of the SICDataScanInterval values be <= 61 // If the interval is <=9, then the interval is stored as a number // For intervals between 10 and 35, uses letters A to Z // For intervals between 36 and 61, uses letters A to Z lastGoodLoc = "Populate scanIntervalList"; var scanIntervalList = string.Empty; if (SICDataScanIntervals != null) { for (var scanIntervalIndex = 0; scanIntervalIndex < sicDetails.SICDataCount; scanIntervalIndex++) { if (SICDataScanIntervals[scanIntervalIndex] <= 9) { scanIntervalList += SICDataScanIntervals[scanIntervalIndex].ToString(); } else if (SICDataScanIntervals[scanIntervalIndex] <= 35) { scanIntervalList += ((char)(SICDataScanIntervals[scanIntervalIndex] + 55)).ToString(); // 55 = -10 + 65 } else if (SICDataScanIntervals[scanIntervalIndex] <= 61) { scanIntervalList += ((char)(SICDataScanIntervals[scanIntervalIndex] + 61)).ToString(); // 61 = -36 + 97 } else { scanIntervalList += "z"; } } } writer.WriteElementString("SICScanIntervals", scanIntervalList); } lastGoodLoc = "Write SICPeakIndexStart"; writer.WriteElementString("SICPeakIndexStart", currentParentIon.SICStats.Peak.IndexBaseLeft.ToString()); writer.WriteElementString("SICPeakIndexEnd", currentParentIon.SICStats.Peak.IndexBaseRight.ToString()); writer.WriteElementString("SICDataCount", sicDetails.SICDataCount.ToString()); if (mOptions.SICOptions.SaveSmoothedData) { writer.WriteElementString("SICSmoothedYDataIndexStart", smoothedYDataSubset.DataStartIndex.ToString()); } if (mOptions.UseBase64DataEncoding) { // Save intensity and mass data lists as base-64 encoded strings // Note that these field names are purposely different than the DataList names used below for comma separated lists lastGoodLoc = "Call SaveDataToXMLEncodeArray with SICIntensityData"; SaveDataToXMLEncodeArray(writer, "SICIntensityData", sicDetails.SICIntensitiesAsFloat); lastGoodLoc = "Call SaveDataToXMLEncodeArray with SICMassData"; SaveDataToXMLEncodeArray(writer, "SICMassData", sicDetails.SICMassesAsFloat); if (mOptions.SICOptions.SaveSmoothedData) { // Need to copy the data into an array with the correct number of elements var dataArray = new float[smoothedYDataSubset.DataCount]; Array.Copy(smoothedYDataSubset.Data, dataArray, smoothedYDataSubset.DataCount); SaveDataToXMLEncodeArray(writer, "SICSmoothedYData", dataArray); } } else { // Save intensity and mass data lists as tab-delimited text list var intensityDataListWritten = false; var massDataList = false; try { lastGoodLoc = "Populate sbIntensityDataList"; sbIntensityDataList.Length = 0; sbMassDataList.Length = 0; if (sicDetails.SICDataCount > 0) { foreach (var dataPoint in sicDetails.SICData) { if (dataPoint.Intensity > 0) { sbIntensityDataList.Append(StringUtilities.DblToString(dataPoint.Intensity, 1) + ","); } else { // Do not output any number if the intensity is 0 sbIntensityDataList.Append(','); } if (dataPoint.Mass > 0) { sbMassDataList.Append(StringUtilities.DblToString(dataPoint.Mass, 3) + ","); } else { // Do not output any number if the mass is 0 sbMassDataList.Append(','); } } // Trim the trailing comma if (sbIntensityDataList[sbIntensityDataList.Length - 1] == ',') { sbIntensityDataList.Length -= 1; sbMassDataList.Length -= 1; } } writer.WriteElementString("IntensityDataList", sbIntensityDataList.ToString()); intensityDataListWritten = true; writer.WriteElementString("MassDataList", sbMassDataList.ToString()); massDataList = true; } catch (OutOfMemoryException ex) { // Ignore the exception if this is an Out of Memory exception if (!intensityDataListWritten) { writer.WriteElementString("IntensityDataList", string.Empty); } if (!massDataList) { writer.WriteElementString("MassDataList", string.Empty); } } if (mOptions.SICOptions.SaveSmoothedData) { try { lastGoodLoc = "Populate sbPeakYDataSmoothed"; sbPeakYDataSmoothed.Length = 0; if (smoothedYDataSubset.Data != null && smoothedYDataSubset.DataCount > 0) { for (var index = 0; index < smoothedYDataSubset.DataCount; index++) { sbPeakYDataSmoothed.Append(Math.Round(smoothedYDataSubset.Data[index]).ToString(CultureInfo.InvariantCulture) + ","); } // Trim the trailing comma sbPeakYDataSmoothed.Length -= 1; } writer.WriteElementString("SmoothedYDataList", sbPeakYDataSmoothed.ToString()); } catch (OutOfMemoryException ex) { // Ignore the exception if this is an Out of Memory exception writer.WriteElementString("SmoothedYDataList", string.Empty); } } } writer.WriteEndElement(); } } catch (Exception ex) { ReportError("Error writing the XML data to the output file; Last good location: " + lastGoodLoc, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } return(true); }
public bool XMLOutputFileInitialize( string inputFilePathFull, string outputDirectoryPath, clsDataOutput dataOutputHandler, clsScanList scanList, clsSpectraCache spectraCache, clsSICOptions sicOptions, clsBinningOptions binningOptions) { var xmlOutputFilePath = string.Empty; try { xmlOutputFilePath = clsDataOutput.ConstructOutputFilePath(inputFilePathFull, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.XMLFile); dataOutputHandler.OutputFileHandles.XMLFileForSICs = new XmlTextWriter(xmlOutputFilePath, System.Text.Encoding.UTF8); var writer = dataOutputHandler.OutputFileHandles.XMLFileForSICs; writer.Formatting = Formatting.Indented; writer.Indentation = 1; writer.WriteStartDocument(true); writer.WriteStartElement("SICData"); writer.WriteStartElement("ProcessingSummary"); writer.WriteElementString("DatasetID", sicOptions.DatasetID.ToString()); writer.WriteElementString("SourceFilePath", inputFilePathFull); string lastModTimeText; string fileSizeBytes; try { var inputFileInfo = new FileInfo(inputFilePathFull); var lastModTime = inputFileInfo.LastWriteTime; lastModTimeText = lastModTime.ToShortDateString() + " " + lastModTime.ToShortTimeString(); fileSizeBytes = inputFileInfo.Length.ToString(); } catch (Exception ex) { lastModTimeText = string.Empty; fileSizeBytes = "0"; } writer.WriteElementString("SourceFileDateTime", lastModTimeText); writer.WriteElementString("SourceFileSizeBytes", fileSizeBytes); writer.WriteElementString("MASICProcessingDate", DateTime.Now.ToString(clsDatasetStatsSummarizer.DATE_TIME_FORMAT_STRING)); writer.WriteElementString("MASICVersion", mOptions.MASICVersion); writer.WriteElementString("MASICPeakFinderDllVersion", mOptions.PeakFinderVersion); writer.WriteElementString("ScanCountTotal", scanList.MasterScanOrderCount.ToString()); writer.WriteElementString("SurveyScanCount", scanList.SurveyScans.Count.ToString()); writer.WriteElementString("FragScanCount", scanList.FragScans.Count.ToString()); writer.WriteElementString("SkipMSMSProcessing", mOptions.SkipMSMSProcessing.ToString()); writer.WriteElementString("ParentIonDecoyMassDa", mOptions.ParentIonDecoyMassDa.ToString("0.0000")); writer.WriteEndElement(); writer.WriteStartElement("MemoryOptions"); writer.WriteElementString("CacheAlwaysDisabled", spectraCache.DiskCachingAlwaysDisabled.ToString()); writer.WriteElementString("CacheSpectraToRetainInMemory", spectraCache.CacheSpectraToRetainInMemory.ToString()); writer.WriteEndElement(); writer.WriteStartElement("SICOptions"); // SIC Options // "SICToleranceDa" is a legacy parameter; If the SIC tolerance is in PPM, then "SICToleranceDa" is the Da tolerance at 1000 m/z writer.WriteElementString("SICToleranceDa", clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, 1000).ToString("0.0000")); writer.WriteElementString("SICTolerance", sicOptions.SICTolerance.ToString("0.0000")); writer.WriteElementString("SICToleranceIsPPM", sicOptions.SICToleranceIsPPM.ToString()); writer.WriteElementString("RefineReportedParentIonMZ", sicOptions.RefineReportedParentIonMZ.ToString()); writer.WriteElementString("ScanRangeStart", sicOptions.ScanRangeStart.ToString()); writer.WriteElementString("ScanRangeEnd", sicOptions.ScanRangeEnd.ToString()); writer.WriteElementString("RTRangeStart", sicOptions.RTRangeStart.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("RTRangeEnd", sicOptions.RTRangeEnd.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("CompressMSSpectraData", sicOptions.CompressMSSpectraData.ToString()); writer.WriteElementString("CompressMSMSSpectraData", sicOptions.CompressMSMSSpectraData.ToString()); writer.WriteElementString("CompressToleranceDivisorForDa", sicOptions.CompressToleranceDivisorForDa.ToString("0.0")); writer.WriteElementString("CompressToleranceDivisorForPPM", sicOptions.CompressToleranceDivisorForPPM.ToString("0.0")); writer.WriteElementString("MaxSICPeakWidthMinutesBackward", sicOptions.MaxSICPeakWidthMinutesBackward.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("MaxSICPeakWidthMinutesForward", sicOptions.MaxSICPeakWidthMinutesForward.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("IntensityThresholdFractionMax", StringUtilities.DblToString(sicOptions.SICPeakFinderOptions.IntensityThresholdFractionMax, 5)); writer.WriteElementString("IntensityThresholdAbsoluteMinimum", sicOptions.SICPeakFinderOptions.IntensityThresholdAbsoluteMinimum.ToString(CultureInfo.InvariantCulture)); // Peak Finding Options var baselineNoiseOptions = sicOptions.SICPeakFinderOptions.SICBaselineNoiseOptions; writer.WriteElementString("SICNoiseThresholdMode", baselineNoiseOptions.BaselineNoiseMode.ToString()); writer.WriteElementString("SICNoiseThresholdIntensity", baselineNoiseOptions.BaselineNoiseLevelAbsolute.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("SICNoiseFractionLowIntensityDataToAverage", StringUtilities.DblToString(baselineNoiseOptions.TrimmedMeanFractionLowIntensityDataToAverage, 5)); writer.WriteElementString("SICNoiseMinimumSignalToNoiseRatio", baselineNoiseOptions.MinimumSignalToNoiseRatio.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("MaxDistanceScansNoOverlap", sicOptions.SICPeakFinderOptions.MaxDistanceScansNoOverlap.ToString()); writer.WriteElementString("MaxAllowedUpwardSpikeFractionMax", StringUtilities.DblToString(sicOptions.SICPeakFinderOptions.MaxAllowedUpwardSpikeFractionMax, 5)); writer.WriteElementString("InitialPeakWidthScansScaler", sicOptions.SICPeakFinderOptions.InitialPeakWidthScansScaler.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("InitialPeakWidthScansMaximum", sicOptions.SICPeakFinderOptions.InitialPeakWidthScansMaximum.ToString()); writer.WriteElementString("FindPeaksOnSmoothedData", sicOptions.SICPeakFinderOptions.FindPeaksOnSmoothedData.ToString()); writer.WriteElementString("SmoothDataRegardlessOfMinimumPeakWidth", sicOptions.SICPeakFinderOptions.SmoothDataRegardlessOfMinimumPeakWidth.ToString()); writer.WriteElementString("UseButterworthSmooth", sicOptions.SICPeakFinderOptions.UseButterworthSmooth.ToString()); writer.WriteElementString("ButterworthSamplingFrequency", StringUtilities.DblToString(sicOptions.SICPeakFinderOptions.ButterworthSamplingFrequency, 5)); writer.WriteElementString("ButterworthSamplingFrequencyDoubledForSIMData", sicOptions.SICPeakFinderOptions.ButterworthSamplingFrequencyDoubledForSIMData.ToString()); writer.WriteElementString("UseSavitzkyGolaySmooth", sicOptions.SICPeakFinderOptions.UseSavitzkyGolaySmooth.ToString()); writer.WriteElementString("SavitzkyGolayFilterOrder", sicOptions.SICPeakFinderOptions.SavitzkyGolayFilterOrder.ToString()); var noiseThresholdOptions = sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions; writer.WriteElementString("MassSpectraNoiseThresholdMode", noiseThresholdOptions.BaselineNoiseMode.ToString()); writer.WriteElementString("MassSpectraNoiseThresholdIntensity", noiseThresholdOptions.BaselineNoiseLevelAbsolute.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("MassSpectraNoiseFractionLowIntensityDataToAverage", StringUtilities.DblToString(noiseThresholdOptions.TrimmedMeanFractionLowIntensityDataToAverage, 5)); writer.WriteElementString("MassSpectraNoiseMinimumSignalToNoiseRatio", noiseThresholdOptions.MinimumSignalToNoiseRatio.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("ReplaceSICZeroesWithMinimumPositiveValueFromMSData", sicOptions.ReplaceSICZeroesWithMinimumPositiveValueFromMSData.ToString()); writer.WriteElementString("SaveSmoothedData", sicOptions.SaveSmoothedData.ToString()); // Similarity options writer.WriteElementString("SimilarIonMZToleranceHalfWidth", sicOptions.SimilarIonMZToleranceHalfWidth.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("SimilarIonToleranceHalfWidthMinutes", sicOptions.SimilarIonToleranceHalfWidthMinutes.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("SpectrumSimilarityMinimum", sicOptions.SpectrumSimilarityMinimum.ToString(CultureInfo.InvariantCulture)); writer.WriteEndElement(); writer.WriteStartElement("BinningOptions"); writer.WriteElementString("BinStartX", binningOptions.StartX.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("BinEndX", binningOptions.EndX.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("BinSize", binningOptions.BinSize.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("MaximumBinCount", binningOptions.MaximumBinCount.ToString()); writer.WriteElementString("IntensityPrecisionPercent", binningOptions.IntensityPrecisionPercent.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("Normalize", binningOptions.Normalize.ToString()); writer.WriteElementString("SumAllIntensitiesForBin", binningOptions.SumAllIntensitiesForBin.ToString()); writer.WriteEndElement(); writer.WriteStartElement("CustomSICValues"); writer.WriteElementString("MZList", mOptions.CustomSICList.RawTextMZList); writer.WriteElementString("MZToleranceDaList", CheckForEmptyToleranceList(mOptions.CustomSICList.RawTextMZToleranceDaList)); writer.WriteElementString("ScanCenterList", mOptions.CustomSICList.RawTextScanOrAcqTimeCenterList); writer.WriteElementString("ScanToleranceList", CheckForEmptyToleranceList(mOptions.CustomSICList.RawTextScanOrAcqTimeToleranceList)); writer.WriteElementString("ScanTolerance", mOptions.CustomSICList.ScanOrAcqTimeTolerance.ToString(CultureInfo.InvariantCulture)); writer.WriteElementString("ScanType", mOptions.CustomSICList.ScanToleranceType.ToString()); writer.WriteElementString("LimitSearchToCustomMZList", mOptions.CustomSICList.LimitSearchToCustomMZList.ToString()); writer.WriteEndElement(); } catch (Exception ex) { ReportError("Error initializing the XML output file: " + xmlOutputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } return(true); }
private bool ExtractXcaliburSurveyScan( XRawFileIO xcaliburAccessor, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, clsSICOptions sicOptions, ThermoRawFileReader.clsScanInfo thermoScanInfo) { var scanInfo = new clsScanInfo() { ScanNumber = thermoScanInfo.ScanNumber, ScanTime = (float)thermoScanInfo.RetentionTime, ScanHeaderText = XRawFileIO.MakeGenericThermoScanFilter(thermoScanInfo.FilterText), ScanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(thermoScanInfo.FilterText), BasePeakIonMZ = thermoScanInfo.BasePeakMZ, BasePeakIonIntensity = thermoScanInfo.BasePeakIntensity, TotalIonIntensity = thermoScanInfo.TotalIonCurrent, MinimumPositiveIntensity = 0, // This will be determined in LoadSpectraForThermoRawFile ZoomScan = thermoScanInfo.ZoomScan, SIMScan = thermoScanInfo.SIMScan, MRMScanType = thermoScanInfo.MRMScanType, LowMass = thermoScanInfo.LowMass, HighMass = thermoScanInfo.HighMass, IsFTMS = thermoScanInfo.IsFTMS }; // Survey scans typically lead to multiple parent ions; we do not record them here scanInfo.FragScanInfo.ParentIonInfoIndex = -1; if (scanInfo.MRMScanType != MRMScanTypeConstants.NotMRM) { // This is an MRM scan scanList.MRMDataPresent = true; } if (scanInfo.SIMScan) { scanList.SIMDataPresent = true; var simKey = scanInfo.LowMass + "_" + scanInfo.HighMass; if (mSIMScanMapping.TryGetValue(simKey, out var simIndex)) { scanInfo.SIMIndex = simIndex; } else { scanInfo.SIMIndex = mSIMScanMapping.Count; mSIMScanMapping.Add(simKey, mSIMScanMapping.Count); } } // Store the ScanEvent values in .ExtendedHeaderInfo StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.ScanEvents); // Store the collision mode and possibly the scan filter text scanInfo.FragScanInfo.CollisionMode = thermoScanInfo.CollisionMode; StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_COLLISION_MODE, thermoScanInfo.CollisionMode); if (mOptions.WriteExtendedStatsIncludeScanFilterText) { StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_SCAN_FILTER_TEXT, thermoScanInfo.FilterText); } if (mOptions.WriteExtendedStatsStatusLog) { // Store the StatusLog values in .ExtendedHeaderInfo StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.StatusLog, mOptions.StatusLogKeyNameFilterList); } scanList.SurveyScans.Add(scanInfo); if (!scanInfo.ZoomScan) { mLastNonZoomSurveyScanIndex = scanList.SurveyScans.Count - 1; } scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.SurveyScan, scanList.SurveyScans.Count - 1); double msDataResolution; if (sicOptions.SICToleranceIsPPM) { // Define MSDataResolution based on the tolerance value that will be used at the lowest m/z in this spectrum, divided by sicOptions.CompressToleranceDivisorForPPM // However, if the lowest m/z value is < 100, then use 100 m/z if (thermoScanInfo.LowMass < 100) { msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, 100) / sicOptions.CompressToleranceDivisorForPPM; } else { msDataResolution = clsParentIonProcessing.GetParentIonToleranceDa(sicOptions, thermoScanInfo.LowMass) / sicOptions.CompressToleranceDivisorForPPM; } } else { msDataResolution = sicOptions.SICTolerance / sicOptions.CompressToleranceDivisorForDa; } // Note: Even if mKeepRawSpectra = False, we still need to load the raw data so that we can compute the noise level for the spectrum var success = LoadSpectraForThermoRawFile( xcaliburAccessor, spectraCache, scanInfo, sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions, clsMASIC.DISCARD_LOW_INTENSITY_MS_DATA_ON_LOAD, sicOptions.CompressMSSpectraData, msDataResolution, mKeepRawSpectra); if (!success) { return(false); } SaveScanStatEntry(dataOutputHandler.OutputFileHandles.ScanStats, clsScanList.eScanTypeConstants.SurveyScan, scanInfo, sicOptions.DatasetID); return(true); }
/// <summary> /// Writes out a flat file containing identified peaks and statistics /// </summary> /// <param name="scanList"></param> /// <param name="inputFileName"></param> /// <param name="outputDirectoryPath"></param> /// <param name="masicOptions"></param> /// <param name="dataOutputHandler"></param> /// <returns></returns> public bool SaveSICStatsFlatFile( clsScanList scanList, string inputFileName, string outputDirectoryPath, clsMASICOptions masicOptions, clsDataOutput dataOutputHandler) { var outputFilePath = string.Empty; const char TAB_DELIMITER = '\t'; // Old: Populate scanListArray with the scan numbers in scanList.SurveyScans // PopulateScanListPointerArray(scanList.SurveyScans, scanList.SurveyScans.Count, out var scanListArray); try { UpdateProgress(0, "Saving SIC data to flat file"); outputFilePath = clsDataOutput.ConstructOutputFilePath(inputFileName, outputDirectoryPath, clsDataOutput.eOutputFileTypeConstants.SICStatsFlatFile); ReportMessage("Saving SIC flat file to disk: " + Path.GetFileName(outputFilePath)); using (var writer = new StreamWriter(outputFilePath, false)) { // Write the SIC stats to the output file // The file is tab delimited var includeScanTimesInSICStatsFile = masicOptions.IncludeScanTimesInSICStatsFile; if (masicOptions.IncludeHeadersInExportFile) { writer.WriteLine(dataOutputHandler.GetHeadersForOutputFile(scanList, clsDataOutput.eOutputFileTypeConstants.SICStatsFlatFile, TAB_DELIMITER)); } if (scanList.SurveyScans.Count == 0 && scanList.ParentIons.Count == 0) { // Write out fake values to the _SICStats.txt file so that downstream software can still access some of the information for (var fragScanIndex = 0; fragScanIndex < scanList.FragScans.Count; fragScanIndex++) { var fakeParentIon = GetFakeParentIonForFragScan(scanList, fragScanIndex); var parentIonIndex = 0; var surveyScanNumber = 0; float surveyScanTime = 0; WriteSICStatsFlatFileEntry(writer, TAB_DELIMITER, masicOptions.SICOptions, scanList, fakeParentIon, parentIonIndex, surveyScanNumber, surveyScanTime, 0, includeScanTimesInSICStatsFile); } } else { for (var parentIonIndex = 0; parentIonIndex < scanList.ParentIons.Count; parentIonIndex++) { bool includeParentIon; if (masicOptions.CustomSICList.LimitSearchToCustomMZList) { includeParentIon = scanList.ParentIons[parentIonIndex].CustomSICPeak; } else { includeParentIon = true; } if (includeParentIon) { for (var fragScanIndex = 0; fragScanIndex < scanList.ParentIons[parentIonIndex].FragScanIndices.Count; fragScanIndex++) { var parentIon = scanList.ParentIons[parentIonIndex]; int surveyScanNumber; float surveyScanTime; if (parentIon.SurveyScanIndex >= 0 && parentIon.SurveyScanIndex < scanList.SurveyScans.Count) { surveyScanNumber = scanList.SurveyScans[parentIon.SurveyScanIndex].ScanNumber; surveyScanTime = scanList.SurveyScans[parentIon.SurveyScanIndex].ScanTime; } else { surveyScanNumber = -1; surveyScanTime = 0; } WriteSICStatsFlatFileEntry(writer, TAB_DELIMITER, masicOptions.SICOptions, scanList, parentIon, parentIonIndex, surveyScanNumber, surveyScanTime, fragScanIndex, includeScanTimesInSICStatsFile); } } if (scanList.ParentIons.Count > 1) { if (parentIonIndex % 100 == 0) { UpdateProgress((short)(parentIonIndex / (double)(scanList.ParentIons.Count - 1) * 100)); } } else { UpdateProgress(1); } if (masicOptions.AbortProcessing) { scanList.ProcessingIncomplete = true; break; } } } } } catch (Exception ex) { Console.WriteLine(ex.StackTrace); ReportError("Error writing the Peak Stats to: " + outputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError); return(false); } return(true); }
private clsParentIonInfo GetFakeParentIonForFragScan(clsScanList scanList, int fragScanIndex) { var currentFragScan = scanList.FragScans[fragScanIndex]; var newParentIon = new clsParentIonInfo(currentFragScan.BasePeakIonMZ) { SurveyScanIndex = 0 }; // Find the previous MS1 scan that occurs before the frag scan var surveyScanNumberAbsolute = currentFragScan.ScanNumber - 1; newParentIon.FragScanIndices.Add(fragScanIndex); if (scanList.MasterScanOrderCount > 0) { var surveyScanIndexMatch = clsBinarySearch.BinarySearchFindNearest( scanList.MasterScanNumList, surveyScanNumberAbsolute, clsBinarySearch.eMissingDataModeConstants.ReturnClosestPoint); while (surveyScanIndexMatch >= 0 && scanList.MasterScanOrder[surveyScanIndexMatch].ScanType == clsScanList.eScanTypeConstants.FragScan) { surveyScanIndexMatch -= 1; } if (surveyScanIndexMatch < 0) { // Did not find the previous survey scan; find the next survey scan surveyScanIndexMatch += 1; while (surveyScanIndexMatch < scanList.MasterScanOrderCount && scanList.MasterScanOrder[surveyScanIndexMatch].ScanType == clsScanList.eScanTypeConstants.FragScan) { surveyScanIndexMatch += 1; } if (surveyScanIndexMatch >= scanList.MasterScanOrderCount) { surveyScanIndexMatch = 0; } } newParentIon.SurveyScanIndex = scanList.MasterScanOrder[surveyScanIndexMatch].ScanIndexPointer; } if (newParentIon.SurveyScanIndex < scanList.SurveyScans.Count) { newParentIon.OptimalPeakApexScanNumber = scanList.SurveyScans[newParentIon.SurveyScanIndex].ScanNumber; } else { newParentIon.OptimalPeakApexScanNumber = surveyScanNumberAbsolute; } newParentIon.PeakApexOverrideParentIonIndex = -1; newParentIon.SICStats.ScanTypeForPeakIndices = clsScanList.eScanTypeConstants.FragScan; newParentIon.SICStats.PeakScanIndexStart = fragScanIndex; newParentIon.SICStats.PeakScanIndexEnd = fragScanIndex; newParentIon.SICStats.PeakScanIndexMax = fragScanIndex; var peak = newParentIon.SICStats.Peak; peak.MaxIntensityValue = currentFragScan.BasePeakIonIntensity; peak.SignalToNoiseRatio = 1; peak.FWHMScanWidth = 1; peak.Area = currentFragScan.BasePeakIonIntensity; peak.ParentIonIntensity = currentFragScan.BasePeakIonIntensity; return(newParentIon); }
private bool ExtractXcaliburFragmentationScan( XRawFileIO xcaliburAccessor, clsScanList scanList, clsSpectraCache spectraCache, clsDataOutput dataOutputHandler, clsSICOptions sicOptions, clsBinningOptions binningOptions, ThermoRawFileReader.clsScanInfo thermoScanInfo) { // Note that MinimumPositiveIntensity will be determined in LoadSpectraForThermoRawFile var scanInfo = new clsScanInfo(thermoScanInfo.ParentIonMZ) { ScanNumber = thermoScanInfo.ScanNumber, ScanTime = (float)thermoScanInfo.RetentionTime, ScanHeaderText = XRawFileIO.MakeGenericThermoScanFilter(thermoScanInfo.FilterText), ScanTypeName = XRawFileIO.GetScanTypeNameFromThermoScanFilterText(thermoScanInfo.FilterText), BasePeakIonMZ = thermoScanInfo.BasePeakMZ, BasePeakIonIntensity = thermoScanInfo.BasePeakIntensity, TotalIonIntensity = thermoScanInfo.TotalIonCurrent, MinimumPositiveIntensity = 0, ZoomScan = thermoScanInfo.ZoomScan, SIMScan = thermoScanInfo.SIMScan, MRMScanType = thermoScanInfo.MRMScanType }; // Typically .EventNumber is 1 for the parent-ion scan; 2 for 1st frag scan, 3 for 2nd frag scan, etc. // This resets for each new parent-ion scan scanInfo.FragScanInfo.FragScanNumber = thermoScanInfo.EventNumber - 1; // The .EventNumber value is sometimes wrong; need to check for this // For example, if the dataset only has MS2 scans and no parent-ion scan, .EventNumber will be 2 for every MS2 scan if (scanList.FragScans.Count > 0) { var prevFragScan = scanList.FragScans[scanList.FragScans.Count - 1]; if (prevFragScan.ScanNumber == scanInfo.ScanNumber - 1) { if (scanInfo.FragScanInfo.FragScanNumber <= prevFragScan.FragScanInfo.FragScanNumber) { scanInfo.FragScanInfo.FragScanNumber = prevFragScan.FragScanInfo.FragScanNumber + 1; } } } scanInfo.FragScanInfo.MSLevel = thermoScanInfo.MSLevel; if (scanInfo.MRMScanType != MRMScanTypeConstants.NotMRM) { // This is an MRM scan scanList.MRMDataPresent = true; scanInfo.MRMScanInfo = clsMRMProcessing.DuplicateMRMInfo(thermoScanInfo.MRMInfo, thermoScanInfo.ParentIonMZ); if (scanList.SurveyScans.Count == 0) { // Need to add a "fake" survey scan that we can map this parent ion to mLastNonZoomSurveyScanIndex = scanList.AddFakeSurveyScan(); } } else { scanInfo.MRMScanInfo.MRMMassCount = 0; } scanInfo.LowMass = thermoScanInfo.LowMass; scanInfo.HighMass = thermoScanInfo.HighMass; scanInfo.IsFTMS = thermoScanInfo.IsFTMS; // Store the ScanEvent values in .ExtendedHeaderInfo StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.ScanEvents); // Store the collision mode and possibly the scan filter text scanInfo.FragScanInfo.CollisionMode = thermoScanInfo.CollisionMode; StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_COLLISION_MODE, thermoScanInfo.CollisionMode); if (mOptions.WriteExtendedStatsIncludeScanFilterText) { StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, clsExtendedStatsWriter.EXTENDED_STATS_HEADER_SCAN_FILTER_TEXT, thermoScanInfo.FilterText); } if (mOptions.WriteExtendedStatsStatusLog) { // Store the StatusLog values in .ExtendedHeaderInfo StoreExtendedHeaderInfo(dataOutputHandler, scanInfo, thermoScanInfo.StatusLog, mOptions.StatusLogKeyNameFilterList); } scanList.FragScans.Add(scanInfo); var fragScanIndex = scanList.FragScans.Count - 1; scanList.AddMasterScanEntry(clsScanList.eScanTypeConstants.FragScan, fragScanIndex); // Note: Even if keepRawSpectra = False, we still need to load the raw data so that we can compute the noise level for the spectrum var msDataResolution = binningOptions.BinSize / sicOptions.CompressToleranceDivisorForDa; var success = LoadSpectraForThermoRawFile( xcaliburAccessor, spectraCache, scanInfo, sicOptions.SICPeakFinderOptions.MassSpectraNoiseThresholdOptions, clsMASIC.DISCARD_LOW_INTENSITY_MSMS_DATA_ON_LOAD, sicOptions.CompressMSMSSpectraData, msDataResolution, mKeepRawSpectra && mKeepMSMSSpectra); if (!success) { return(false); } SaveScanStatEntry(dataOutputHandler.OutputFileHandles.ScanStats, clsScanList.eScanTypeConstants.FragScan, scanInfo, sicOptions.DatasetID); if (thermoScanInfo.MRMScanType == MRMScanTypeConstants.NotMRM) { // This is not an MRM scan mParentIonProcessor.AddUpdateParentIons(scanList, mLastNonZoomSurveyScanIndex, thermoScanInfo.ParentIonMZ, fragScanIndex, spectraCache, sicOptions); } else { // This is an MRM scan mParentIonProcessor.AddUpdateParentIons(scanList, mLastNonZoomSurveyScanIndex, thermoScanInfo.ParentIonMZ, scanInfo.MRMScanInfo, spectraCache, sicOptions); } if (mLastNonZoomSurveyScanIndex >= 0) { var precursorScanNumber = scanList.SurveyScans[mLastNonZoomSurveyScanIndex].ScanNumber; // Compute the interference of the parent ion in the MS1 spectrum for this frag scan scanInfo.FragScanInfo.InterferenceScore = ComputeInterference(xcaliburAccessor, thermoScanInfo, precursorScanNumber); } return(true); }