private bool ExtractScanInfoWork(
XRawFileIO xcaliburAccessor,
clsScanList scanList,
clsSpectraCache spectraCache,
clsDataOutput dataOutputHandler,
clsSICOptions sicOptions,
ThermoRawFileReader.clsScanInfo thermoScanInfo)
{
if (thermoScanInfo.ParentIonMZ > 0 && Math.Abs(mOptions.ParentIonDecoyMassDa) > 0)
{
thermoScanInfo.ParentIonMZ += mOptions.ParentIonDecoyMassDa;
}
bool success;
// Determine if this was an MS/MS scan
// If yes, determine the scan number of the survey scan
if (thermoScanInfo.MSLevel <= 1)
{
// Survey Scan
success = ExtractXcaliburSurveyScan(xcaliburAccessor, scanList, spectraCache, dataOutputHandler,
sicOptions, thermoScanInfo);
}
else
{
// Fragmentation Scan
success = ExtractXcaliburFragmentationScan(xcaliburAccessor, scanList, spectraCache, dataOutputHandler,
sicOptions, mOptions.BinningOptions, thermoScanInfo);
}
return(success);
}
private static void ShowMethod(XRawFileIO reader)
{
try
{
foreach (var method in reader.FileInfo.InstMethods)
{
Console.WriteLine();
Console.WriteLine("Instrument model: " + reader.FileInfo.InstModel);
Console.WriteLine("Instrument name: " + reader.FileInfo.InstName);
Console.WriteLine("Instrument description: " + reader.FileInfo.InstrumentDescription);
Console.WriteLine("Instrument serial number: " + reader.FileInfo.InstSerialNumber);
Console.WriteLine();
if (string.IsNullOrWhiteSpace(method))
{
continue;
}
if (method.Length > 500)
{
Console.WriteLine(method.Substring(0, 500) + " ...");
}
else
{
Console.WriteLine(method);
}
}
}
catch (Exception ex)
{
ShowError("Error loading the MS Method: " + ex.Message, ex);
}
}
/// <summary>
/// Constructor - open the file, and prepare to read
/// </summary>
/// <param name="filePath"></param>
public XCaliburReader(string filePath)
{
_cachedScanInfo = new clsScanInfo(-1);
_msfileReader = new XRawFileIO();
var dataFile = new FileInfo(filePath);
if (!dataFile.Exists)
{
throw new FileNotFoundException("Thermo .raw file not found: " + filePath, dataFile.FullName);
}
FilePath = filePath;
using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (var sha1 = new SHA1Managed())
{
var hash = sha1.ComputeHash(fs);
SrcFileChecksum = BitConverter.ToString(hash).ToLower().Replace("-", "");
}
_msfileReader.OpenRawFile(filePath);
_minLcScan = 1;
_numSpectra = ReadNumSpectra();
_maxLcScan = _numSpectra;
_msLevel = new Dictionary <int, int>();
FileFormatVersion = _msfileReader.FileInfo.VersionNumber.ToString();
}
private string GetMzAnalyzer(XRawFileIO reader)
{
var startScan = reader.ScanStart;
var success = reader.GetScanInfo(startScan, out var scanInfo);
if (!success)
{
OnWarningEvent(string.Format("Scan {0} not found in GetIonizationSource", startScan));
return("Unknown");
}
foreach (var param in scanInfo.FilterText.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries))
{
if (param.Equals("FTMS"))
{
return(param);
}
if (param.Equals("ITMS"))
{
return(param);
}
}
OnWarningEvent("Unrecognized MzAnalyzer; filter line does not contain FTMS or ITMS: " + scanInfo.FilterText);
return("Unknown");
}
private string WriteMsRunTag(XRawFileIO reader, IReadOnlyCollection <int> targetScans)
{
if (targetScans.Count == 0)
{
OnWarningEvent("targetScans sent to WriteMsRunTag is empty; cannot create a valid .mzXML file");
return(string.Empty);
}
var startScanFound = reader.GetScanInfo(targetScans.First(), out var scanFirst);
if (!startScanFound)
{
OnWarningEvent(string.Format("Unable to find scan {0} in WriteMsRunTag", targetScans.First()));
return(string.Empty);
}
var endScanFound = reader.GetScanInfo(targetScans.Last(), out var scanLast);
if (!endScanFound)
{
OnWarningEvent(string.Format("Unable to find scan {0} in WriteMsRunTag", targetScans.Last()));
return(string.Empty);
}
var returnString = " <msRun scanCount=\"" + targetScans.Count + "\" startTime=\"";
var startTime = "PT" + Math.Round(scanFirst.RetentionTime * 60, 8) + "S\" ";
var endTime = "endTime=\"PT" + Math.Round(scanLast.RetentionTime * 60, 8) + "S\">";
return(returnString + startTime + endTime);
}
/// <summary>
/// This function is used to determine one or more overall quality scores
/// </summary>
/// <param name="objXcaliburAccessor"></param>
/// <param name="datasetFileInfo"></param>
/// <remarks></remarks>
private void ComputeQualityScores(XRawFileIO objXcaliburAccessor, clsDatasetFileInfo datasetFileInfo)
{
float sngOverallScore;
double dblOverallAvgIntensitySum = 0;
var intOverallAvgCount = 0;
if (mLCMS2DPlot.ScanCountCached > 0)
{
// Obtain the overall average intensity value using the data cached in mLCMS2DPlot
// This avoids having to reload all of the data using objXcaliburAccessor
const int intMSLevelFilter = 1;
sngOverallScore = mLCMS2DPlot.ComputeAverageIntensityAllScans(intMSLevelFilter);
}
else
{
var intScanCount = objXcaliburAccessor.GetNumScans();
GetStartAndEndScans(intScanCount, out var intScanStart, out var intScanEnd);
for (var intScanNumber = intScanStart; intScanNumber <= intScanEnd; intScanNumber++)
{
// This function returns the number of points in dblMassIntensityPairs()
var intReturnCode = objXcaliburAccessor.GetScanData2D(intScanNumber, out var dblMassIntensityPairs);
if (intReturnCode <= 0)
{
continue;
}
if ((dblMassIntensityPairs == null) || dblMassIntensityPairs.GetLength(1) <= 0)
{
continue;
}
// Keep track of the quality scores and then store one or more overall quality scores in datasetFileInfo.OverallQualityScore
// For now, this just computes the average intensity for each scan and then computes and overall average intensity value
double dblIntensitySum = 0;
for (var intIonIndex = 0; intIonIndex <= dblMassIntensityPairs.GetUpperBound(1); intIonIndex++)
{
dblIntensitySum += dblMassIntensityPairs[1, intIonIndex];
}
dblOverallAvgIntensitySum += dblIntensitySum / dblMassIntensityPairs.GetLength(1);
intOverallAvgCount += 1;
}
if (intOverallAvgCount > 0)
{
sngOverallScore = (float)(dblOverallAvgIntensitySum / intOverallAvgCount);
}
else
{
sngOverallScore = 0;
}
}
datasetFileInfo.OverallQualityScore = sngOverallScore;
}
private int GetScanData(XRawFileIO reader, clsScanInfo scanInfo, out double[] mzList, out double[] intensityList)
{
if (scanInfo.IsFTMS)
{
var labelDataCount = reader.GetScanLabelData(scanInfo.ScanNumber, out var labelData);
if (labelDataCount > 0)
{
mzList = new double[labelDataCount];
intensityList = new double[labelDataCount];
for (var i = 0; i < labelDataCount; i++)
{
mzList[i] = labelData[i].Mass;
intensityList[i] = labelData[i].Intensity;
}
return(labelDataCount);
}
}
var dataPointCount = reader.GetScanData(scanInfo.ScanNumber, out mzList, out intensityList, 0, true);
return(dataPointCount);
}
/// <summary>
/// The constructor of this parser will attempt to open any files that
/// end in ".RAW". If a null string is passed in or a
/// a string without a ".raw" file extension then an exception
/// will be thrown.
/// </summary>
/// <param name="fileLocation">This must be a file Location to a ".raw" file.</param>
public ThermoRawParser(string filelocation)
{
this.m_fileLocation = filelocation;
string extension = Path.GetExtension(filelocation);
extension = extension.ToLower();
if (m_fileLocation != null)
{
if (extension == ".raw")
{
m_fileToRead = new XRawFileIO();
m_fileOpened = m_fileToRead.OpenRawFile(m_fileLocation); //m_fileOpened is staying false for some reason. ****
}
else
{
throw new System.InvalidProgramException("Invalid File Type");
}
}
else
{
throw new System.InvalidOperationException("The file location passed was equal to null");
}
}
private string GetIonizationSource(XRawFileIO reader)
{
var startScan = reader.ScanStart;
var success = reader.GetScanInfo(startScan, out var scanInfo);
if (!success)
{
OnWarningEvent(string.Format("Scan {0} not found in GetIonizationSource", startScan));
return("Unknown");
}
var filterLineParts = scanInfo.FilterText.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var param in filterLineParts)
{
if (param.Equals("NSI"))
{
return(param);
}
if (param.Equals("ESI"))
{
return(param);
}
}
OnWarningEvent("Unrecognized ionization source; filter line does not contain NSI or ESI: " + scanInfo.FilterText);
return("Unknown");
}
private bool GetCvValue(XRawFileIO reader, int scanNumber, out float cvValue, out string filterTextMatch, bool showWarnings = false)
{
cvValue = 0;
filterTextMatch = string.Empty;
if (!reader.GetScanInfo(scanNumber, out var scanInfo))
{
if (showWarnings)
{
OnWarningEvent(string.Format("Scan {0} not found; skipping", scanNumber));
}
return(false);
}
var filterText = scanInfo.FilterText;
if (filterText.IndexOf("cv=", StringComparison.OrdinalIgnoreCase) < 0)
{
if (showWarnings)
{
AddToScanWarningList(
reader.RawFilePath, scanNumber,
string.Format("Scan {0} does not contain cv=; skipping", scanNumber));
}
return(false);
}
var match = mCvMatcher.Match(filterText);
if (!match.Success)
{
if (showWarnings)
{
AddToScanWarningList(
reader.RawFilePath, scanNumber,
string.Format(
"Scan {0} has cv= in the filter text, but it is not followed by a number: {1}",
scanNumber, filterText));
}
return(false);
}
if (!float.TryParse(match.Groups["CV"].Value, out cvValue))
{
if (showWarnings)
{
AddToScanWarningList(
reader.RawFilePath, scanNumber,
string.Format(
"Unable to parse the CV value for scan {0}: {1}",
scanNumber, match.Groups["CV"].Value));
}
return(false);
}
filterTextMatch = match.Value;
return(true);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="thermoRawFilePath"></param>
/// <param name="loadTuneInfo"></param>
public XCaliburRun2(string thermoRawFilePath, bool loadTuneInfo = false)
{
Check.Require(File.Exists(thermoRawFilePath), "Run not initialized. File not found: " + thermoRawFilePath);
DatasetFileOrDirectoryPath = thermoRawFilePath;
var baseFilename = Path.GetFileName(DatasetFileOrDirectoryPath);
if (baseFilename == null)
{
throw new FileNotFoundException("Unable to determine the filename for " + thermoRawFilePath);
}
DatasetName = baseFilename.Substring(0, baseFilename.LastIndexOf('.'));
DatasetDirectoryPath = Path.GetDirectoryName(thermoRawFilePath);
var options = new ThermoReaderOptions
{
LoadMSMethodInfo = false,
LoadMSTuneInfo = loadTuneInfo
};
mRawFileReader = new XRawFileIO(DatasetFileOrDirectoryPath, options);
IsDataThresholded = true;
IsMsAbundanceReportedAsAverage = true;
MSFileType = Globals.MSFileType.Thermo_Raw;
ContainsMSMSData = true;
XYData = new XYData();
ParentScanList = new Dictionary <int, int>();
MinLCScan = 1;
MaxLCScan = GetNumMSScans();
}
/// <summary>
/// Constructor - open the file, and prepare to read
/// </summary>
/// <param name="filePath"></param>
public XCaliburReader(string filePath)
{
_cachedScanInfo = new clsScanInfo(-1);
_msfileReader = new XRawFileIO();
_msfileReader.ErrorEvent += _msfileReader_ErrorEvent;
var dataFile = new FileInfo(filePath);
if (!dataFile.Exists)
{
throw new FileNotFoundException("Thermo .raw file not found: " + filePath, dataFile.FullName);
}
FilePath = filePath;
_checkSum = string.Empty;
_msfileReader.OpenRawFile(filePath);
_minLcScan = 1;
NumSpectra = ReadNumSpectra();
_maxLcScan = NumSpectra;
_msLevel = new Dictionary <int, int>();
FileFormatVersion = _msfileReader.FileInfo.VersionNumber.ToString();
}
public MsScan(int scanNumber, XRawFileIO reader)
{
ScanNumber = scanNumber;
if (!reader.GetScanInfo(scanNumber, out var scanInfo))
{
ConsoleMsgUtils.ShowWarning("Scan {0} not found in {1}", scanNumber, Path.GetFileName(reader.RawFilePath));
return;
}
// Get centroided data
var dataPointCount = GetScanData(reader, scanInfo, out var mzList, out var intensityList);
MsLevel = scanInfo.MSLevel;
PeaksCount = dataPointCount;
switch (scanInfo.IonMode)
{
case IonModeConstants.Positive:
Polarity = "+";
break;
case IonModeConstants.Negative:
Polarity = "-";
break;
case IonModeConstants.Unknown:
Polarity = string.Empty;
break;
}
FilterLine = scanInfo.FilterText;
ScanType = GetScanType();
RetentionTime = "PT" + Math.Round(scanInfo.RetentionTime * 60, 8) + "S";
TotIonCurrent = scanInfo.TotalIonCurrent;
string encodedPeaks;
if (PeaksCount == 0)
{
LowMz = 0;
HighMz = 0;
BasePeakMz = 0;
BasePeakIntensity = 0;
encodedPeaks = Base64EncodeMsData(mzList, intensityList);
}
else
{
LowMz = mzList.Min();
HighMz = mzList.Max();
BasePeakMz = scanInfo.BasePeakMZ;
BasePeakIntensity = scanInfo.BasePeakIntensity;
encodedPeaks = Base64EncodeMsData(mzList, intensityList);
}
PeakData = new Peaks(32, "network", "m/z-int", "none", 0, encodedPeaks);
}
private void WriteHeader(TextWriter writer, string filePath, XRawFileIO reader, string hash, IReadOnlyCollection <int> targetScans)
{
var version = ProcessFilesOrDirectoriesBase.GetEntryOrExecutingAssembly().GetName().Version;
var sb = new StringBuilder();
sb.AppendLine("<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>");
sb.AppendLine("<mzXML xmlns=\"http://sashimi.sourceforge.net/schema_revision/mzXML_3.1\"");
sb.AppendLine(" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"");
sb.AppendLine(" xsi:schemaLocation=\"http://sashimi.sourceforge.net/schema_revision/mzXML_3.1 http://sashimi.sourceforge.net/schema_revision/mzXML_3.1/mzXML_idx_3.1.xsd\">");
sb.AppendLine(WriteMsRunTag(reader, targetScans));
sb.AppendLine(WriteParentFileTag(filePath, hash));
sb.AppendLine(" <msInstrument>");
sb.AppendLine(" <msManufacturer category=\"msManufacturer\" value=\"Thermo Finnigan\" />");
sb.AppendLine(" <msModel category=\"msModel\" value=\"unknown\" />");
sb.AppendFormat(" <msIonisation category=\"msIonisation\" value=\"{0}\" />", GetIonizationSource(reader)).AppendLine();
sb.AppendFormat(" <msMassAnalyzer category=\"msMassAnalyzer\" value=\"{0}\" />", GetMzAnalyzer(reader)).AppendLine();
sb.AppendLine(" <msDetector category=\"msDetector\" value=\"unknown\" />");
sb.AppendLine(" <software type=\"acquisition\" name=\"Xcalibur\" version=\"3.1.2279\" />");
sb.AppendLine(" </msInstrument>");
sb.AppendLine(" <dataProcessing centroided=\"1\">");
sb.AppendFormat(
" <software type=\"conversion\" name=\"WriteFaimsXMLFromRawFile\" version=\"{0}\" />", version).AppendLine();
sb.Append(" </dataProcessing>");
var headerText = sb.ToString();
ByteTracker(headerText);
writer.WriteLine(headerText);
}
/// <summary>
/// This function is used to determine one or more overall quality scores
/// </summary>
/// <param name="xcaliburAccessor"></param>
/// <param name="datasetFileInfo"></param>
/// <remarks></remarks>
private void ComputeQualityScores(XRawFileIO xcaliburAccessor, DatasetFileInfo datasetFileInfo)
{
float overallScore;
double overallAvgIntensitySum = 0;
var overallAvgCount = 0;
if (mLCMS2DPlot.ScanCountCached > 0)
{
// Obtain the overall average intensity value using the data cached in mLCMS2DPlot
// This avoids having to reload all of the data using xcaliburAccessor
const int msLevelFilter = 1;
overallScore = mLCMS2DPlot.ComputeAverageIntensityAllScans(msLevelFilter);
}
else
{
var scanCount = xcaliburAccessor.GetNumScans();
GetStartAndEndScans(scanCount, out var scanStart, out var scanEnd);
for (var scanNumber = scanStart; scanNumber <= scanEnd; scanNumber++)
{
// This function returns the number of points in massIntensityPairs()
var returnCode = xcaliburAccessor.GetScanData2D(scanNumber, out var massIntensityPairs);
if (returnCode <= 0)
{
continue;
}
if (massIntensityPairs == null || massIntensityPairs.GetLength(1) <= 0)
{
continue;
}
// Keep track of the quality scores and then store one or more overall quality scores in datasetFileInfo.OverallQualityScore
// For now, this just computes the average intensity for each scan and then computes and overall average intensity value
double intensitySum = 0;
for (var ionIndex = 0; ionIndex <= massIntensityPairs.GetUpperBound(1); ionIndex++)
{
intensitySum += massIntensityPairs[1, ionIndex];
}
overallAvgIntensitySum += intensitySum / massIntensityPairs.GetLength(1);
overallAvgCount += 1;
}
if (overallAvgCount > 0)
{
overallScore = (float)(overallAvgIntensitySum / overallAvgCount);
}
else
{
overallScore = 0;
}
}
datasetFileInfo.OverallQualityScore = overallScore;
}
private void AddThermoDevices(
XRawFileIO xcaliburAccessor,
DatasetFileInfo datasetFileInfo,
ICollection <Device> deviceMatchList,
ICollection <Device> deviceSkipList)
{
// ReSharper disable once ForeachCanBePartlyConvertedToQueryUsingAnotherGetEnumerator
foreach (var device in xcaliburAccessor.FileInfo.Devices)
{
if (deviceMatchList.Count > 0 && !deviceMatchList.Contains(device.Key))
{
continue;
}
if (deviceSkipList.Count > 0 && deviceSkipList.Contains(device.Key))
{
continue;
}
for (var deviceNumber = 1; deviceNumber <= device.Value; deviceNumber++)
{
var deviceInfo = xcaliburAccessor.GetDeviceInfo(device.Key, deviceNumber);
datasetFileInfo.DeviceList.Add(deviceInfo);
}
}
}
private void CloseReader()
{
if (rawFile != null)
{
rawFile.Dispose();
rawFile = null;
}
}
/// <summary>
/// Write the tune method file to disk
/// </summary>
/// <param name="rawFileReader"></param>
/// <param name="dataOutputHandler"></param>
/// <returns></returns>
public bool SaveMSTuneFile(
XRawFileIO rawFileReader,
clsDataOutput dataOutputHandler)
{
const char TAB_DELIMITER = '\t';
int tuneMethodCount;
var outputFilePath = "?UndefinedFile?";
try
{
tuneMethodCount = rawFileReader.FileInfo.TuneMethods.Count;
}
catch (Exception ex)
{
ReportError("Error looking up TuneMethods length in rawFileReader.FileInfo", ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError);
return(false);
}
try
{
for (var index = 0; index < tuneMethodCount; index++)
{
string tuneInfoNum;
if (index == 0 && rawFileReader.FileInfo.TuneMethods.Count == 1)
{
tuneInfoNum = string.Empty;
}
else
{
tuneInfoNum = (index + 1).ToString().Trim();
}
outputFilePath = dataOutputHandler.OutputFileHandles.MSTuneFilePathBase + tuneInfoNum + ".txt";
using (var writer = new StreamWriter(outputFilePath, false))
{
writer.WriteLine("Category" + TAB_DELIMITER + "Name" + TAB_DELIMITER + "Value");
foreach (var setting in rawFileReader.FileInfo.TuneMethods[index].Settings)
{
writer.WriteLine(setting.Category + TAB_DELIMITER + setting.Name + TAB_DELIMITER + setting.Value);
}
writer.WriteLine();
}
}
}
catch (Exception ex)
{
ReportError("Error writing the MS Tune Settings to: " + outputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError);
return(false);
}
return(true);
}
/// <summary>
/// Write the instrument method file to disk
/// </summary>
/// <param name="rawFileReader"></param>
/// <param name="dataOutputHandler"></param>
/// <returns></returns>
public bool SaveMSMethodFile(
XRawFileIO rawFileReader,
clsDataOutput dataOutputHandler)
{
int instMethodCount;
var outputFilePath = "?UndefinedFile?";
try
{
instMethodCount = rawFileReader.FileInfo.InstMethods.Count;
}
catch (Exception ex)
{
ReportError("Error looking up InstMethod length in rawFileReader.FileInfo", ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError);
return(false);
}
try
{
for (var index = 0; index < instMethodCount; index++)
{
string methodNum;
if (index == 0 && rawFileReader.FileInfo.InstMethods.Count == 1)
{
methodNum = string.Empty;
}
else
{
methodNum = (index + 1).ToString().Trim();
}
outputFilePath = dataOutputHandler.OutputFileHandles.MSMethodFilePathBase + methodNum + ".txt";
using (var writer = new StreamWriter(outputFilePath, false))
{
var fileInfo = rawFileReader.FileInfo;
writer.WriteLine("Instrument model: " + fileInfo.InstModel);
writer.WriteLine("Instrument name: " + fileInfo.InstName);
writer.WriteLine("Instrument description: " + fileInfo.InstrumentDescription);
writer.WriteLine("Instrument serial number: " + fileInfo.InstSerialNumber);
writer.WriteLine();
writer.WriteLine(rawFileReader.FileInfo.InstMethods[index]);
}
}
}
catch (Exception ex)
{
ReportError("Error writing the MS Method to: " + outputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError);
return(false);
}
return(true);
}
public MSFileReaderMetadata(string rawFilePath)
{
RawFilePath = rawFilePath;
if (!XRawFileIO.IsMSFileReaderInstalled(out var error))
{
if (!string.IsNullOrWhiteSpace(error))
{
throw new Exception(error);
}
throw new Exception("Cannot create an instance of XRawFile. Check the MSFileReader installation.");
}
rawFile = new XRawFileIO(rawFilePath);
}
public void ReadCollisionEnergyValsTest1()
{
var testFile = @"\\proto-2\UnitTest_Files\DeconTools_TestFiles\QC_Shew_08_04-pt5-2_11Jan09_Sphinx_08-11-18.RAW";
var testScan = 5000;
_thermoFileReader = new ThermoRawFileReader.XRawFileIO(testFile);
_thermoFileReader.GetScanInfo(testScan, out clsScanInfo scanInfo);
Console.WriteLine("RT = " + scanInfo.RetentionTime);
if (scanInfo.TryGetScanEvent("Ion Injection Time (ms):", out var eventValue, true))
{
Console.WriteLine("Ion injection time= " + Convert.ToDouble(eventValue));
}
}
private bool ExtractScanInfoCheckRange(
XRawFileIO xcaliburAccessor,
ThermoRawFileReader.clsScanInfo thermoScanInfo,
clsScanList scanList,
clsSpectraCache spectraCache,
clsDataOutput dataOutputHandler,
double percentComplete)
{
bool success;
if (mScanTracking.CheckScanInRange(thermoScanInfo.ScanNumber, thermoScanInfo.RetentionTime, mOptions.SICOptions))
{
success = ExtractScanInfoWork(xcaliburAccessor, scanList, spectraCache, dataOutputHandler,
mOptions.SICOptions, thermoScanInfo);
}
else
{
mScansOutOfRange += 1;
success = true;
}
UpdateProgress((short)Math.Round(percentComplete, 0));
UpdateCacheStats(spectraCache);
if (mOptions.AbortProcessing)
{
scanList.ProcessingIncomplete = true;
return(false);
}
if (DateTime.UtcNow.Subtract(mLastLogTime).TotalSeconds >= 10 ||
thermoScanInfo.ScanNumber % 500 == 0 && (
thermoScanInfo.ScanNumber >= mOptions.SICOptions.ScanRangeStart &&
thermoScanInfo.ScanNumber <= mOptions.SICOptions.ScanRangeEnd))
{
ReportMessage("Reading scan: " + thermoScanInfo.ScanNumber.ToString());
Console.Write(".");
mLastLogTime = DateTime.UtcNow;
}
return(success);
}
protected bool UpdateDatasetFileStats(
FileInfo rawFileInfo,
int datasetID,
XRawFileIO xcaliburAccessor)
{
// Read the file info from the file system
var success = UpdateDatasetFileStats(rawFileInfo, datasetID);
if (!success)
{
return(false);
}
// Read the file info using the Xcalibur Accessor
try
{
mDatasetFileInfo.AcqTimeStart = xcaliburAccessor.FileInfo.CreationDate;
}
catch (Exception ex)
{
// Read error
return(false);
}
try
{
// Look up the end scan time then compute .AcqTimeEnd
var scanEnd = xcaliburAccessor.ScanEnd;
xcaliburAccessor.GetScanInfo(scanEnd, out ThermoRawFileReader.clsScanInfo scanInfo);
mDatasetFileInfo.AcqTimeEnd = mDatasetFileInfo.AcqTimeStart.AddMinutes(scanInfo.RetentionTime);
mDatasetFileInfo.ScanCount = xcaliburAccessor.GetNumScans();
}
catch (Exception ex)
{
// Error; use default values
mDatasetFileInfo.AcqTimeEnd = mDatasetFileInfo.AcqTimeStart;
mDatasetFileInfo.ScanCount = 0;
}
return(true);
}
public List <TransitionData> GetAllTransitions()
{
var srmTransitions = new List <TransitionData>();
using (var reader = new XRawFileIO(DatasetPath))
{
//Console.WriteLine($"File \"{DatasetPath}\": {reader.FileInfo.InstMethods.Count} instrument methods");
foreach (var method in reader.FileInfo.InstMethods)
{
//Console.WriteLine($"File \"{DatasetPath}\": InstMethod string length: {method.Length}");
var parsed = new XCalInstMethodParser(method);
if (parsed.UsesCompoundName)
{
canUseCompoundNames = true;
}
srmTransitions.AddRange(parsed.ParseSrmTable());
}
}
//Console.WriteLine($"File \"{DatasetPath}\": {srmTransitions.Count} transitions in instrument method");
return(srmTransitions);
}
private clsScanInfo GetParentScan(XRawFileIO reader, clsScanInfo scanInfo)
{
if (scanInfo.MSLevel <= 1)
{
return(scanInfo);
}
var parentScanNumber = scanInfo.ScanNumber - 1;
while (parentScanNumber > 0)
{
var success = reader.GetScanInfo(parentScanNumber, out var parentScanInfo);
if (success && parentScanInfo.MSLevel <= 1)
{
return(parentScanInfo);
}
parentScanNumber--;
}
reader.GetScanInfo(1, out var scanInfoFirstScan);
return(scanInfoFirstScan);
}
private List <int> FindAllTargetScans(double targetCV, XRawFileIO reader)
{
var targetScans = new List <int>();
var startScan = reader.ScanStart;
var endScan = reader.ScanEnd;
for (var scanNumber = startScan; scanNumber <= endScan; scanNumber++)
{
var success = GetCvValue(reader, scanNumber, out var cvValue, out _);
if (!success)
{
continue;
}
if (Math.Abs(cvValue - targetCV) < float.Epsilon)
{
targetScans.Add(scanNumber);
}
}
return(targetScans);
}
private bool ProcessFile(string filePath, string outputDirectoryPath)
{
try
{
OnStatusEvent(string.Format("Processing {0}", filePath));
// Compute the SHA-1 hash of the .Raw file
var fileSha1 = HashRawFile(filePath);
// Disable loading the method
// Method loading doesn't work on Linux and this program doesn't need that information
var readerOptions = new ThermoReaderOptions
{
LoadMSMethodInfo = false
};
// Open up the raw file connection
var reader = new XRawFileIO(filePath, readerOptions);
// Get all unique CV values from scans
var cvValues = GetUniqueCvValues(reader);
var totalScanCount = reader.GetNumScans();
var scansProcessed = 0;
var lastProgress = DateTime.UtcNow;
// Reset tracking variables
ByteTracking.Reset(true);
// Now work for each unique CV value (# files we're going to need to split into)
// get all scans that have the CV that we're currently targeting
foreach (var cvValue in cvValues)
{
var baseName = Path.GetFileNameWithoutExtension(filePath);
var mzXmlPath = Path.Combine(outputDirectoryPath, baseName + "_" + cvValue + ".mzXML");
OnDebugEvent(string.Format("Creating file {0}", mzXmlPath));
var targetScans = FindAllTargetScans(cvValue, reader);
var parentScanToMS2Map = new Dictionary <int, List <int> >();
using (var writer = new StreamWriter(new FileStream(mzXmlPath, FileMode.Create, FileAccess.Write, FileShare.Read)))
{
WriteHeader(writer, filePath, reader, fileSha1, targetScans);
Ms1Scan currentMS1Scan = null;
// Write out our target scans
foreach (var scanNumber in targetScans)
{
if (DateTime.UtcNow.Subtract(lastProgress).TotalSeconds >= 3)
{
var percentComplete = scansProcessed / (double)totalScanCount * 100;
OnDebugEvent(string.Format("... processing: {0:F0}% complete", percentComplete));
lastProgress = DateTime.UtcNow;
}
var scan = new MsScan(scanNumber, reader);
if (scan.MsLevel == 1)
{
parentScanToMS2Map.Add(scan.ScanNumber, new List <int>());
if (currentMS1Scan == null)
{
// Start condition
currentMS1Scan = Ms1Scan.Create(scan);
}
else
{
// Write currentMS1Scan to file
var outString = currentMS1Scan.ToXML(this);
writer.WriteLine(outString);
currentMS1Scan = Ms1Scan.Create(scan);
}
}
else if (scan.MsLevel == 2)
{
if (currentMS1Scan != null)
{
parentScanToMS2Map[currentMS1Scan.ScanNumber].Add(scan.ScanNumber);
}
var ms2Scan = Ms2Scan.Create(scan);
ms2Scan.AddMs2ScanParameters(reader);
currentMS1Scan?.AddMs2Scan(ms2Scan);
}
scansProcessed++;
}
if (currentMS1Scan != null)
{
// Once we're out, we need to write out our last currentMS1Scan
writer.WriteLine(currentMS1Scan.ToXML(this));
}
// Finish off msRun
writer.WriteLine(" </msRun>");
writer.WriteLine(" <index name=\"scan\">");
// Add special entry to our indexOffset list for where the offsets start
var index = new Index(0, ByteTracking.ByteDepth + ByteTracking.Encoder.GetByteCount(" </msRun>") + 3);
ByteTracking.ScanOffsets.Add(index);
// Write all index offsets
for (var i = 0; i < ByteTracking.ScanOffsets.Count - 1; i++)
{
var offset = ByteTracking.ScanOffsets[i];
writer.WriteLine(" <offset id=\"" + offset.ScanNumber + "\">" + offset.ByteDepth + "</offset>");
}
writer.WriteLine(" </index>");
writer.WriteLine(" <indexOffset>" + ByteTracking.ScanOffsets.Last().ByteDepth + "</indexOffset>");
writer.Write(" <sha1>");
}
// Compute the SHA-1 hash of the file up to this point
var mzXmlHash = HashMzXML(mzXmlPath);
// Append the hash
using (var writer = new StreamWriter(mzXmlPath, true))
{
writer.Write(mzXmlHash);
writer.WriteLine("</sha1>");
writer.WriteLine("</mzXML>");
}
// Reset tracking variables, but do not restart scan numbering
ByteTracking.Reset(false);
}
OnDebugEvent(string.Format("... processing: {0:F0}% complete", 100));
return(true);
}
catch (Exception ex)
{
OnErrorEvent("Error in ProcessFile", ex);
return(false);
}
}
private int ValidateScanNumber(int scan, XRawFileIO rawReader)
{
var totalSpectra = rawReader.GetNumScans();
if (scan > totalSpectra)
{
throw new ScanOutOfRangeException("The requested scan is out of range.");
}
return scan;
}
private List<XYData> LoadRawSpectra(XRawFileIO rawReader, int scan)
{
double[] mz;
double[] intensities;
rawReader.GetScanData(scan, out mz, out intensities);
if (mz == null)
return new List<XYData>();
var data = new List<XYData>(mz.Length);
for (var i = 0; i < mz.Length; i++)
{
var intensity = intensities[i];
data.Add(new XYData(mz[i], intensity));
}
return data;
}
private ScanSummary GetScanSummary(int scan, XRawFileIO rawReader, out FinniganFileReaderBaseClass.udtScanHeaderInfoType header)
{
rawReader.GetScanInfo(scan, out header);
var summary = new ScanSummary
{
MsLevel = header.MSLevel,
Time = header.RetentionTime,
Scan = scan,
TotalIonCurrent = Convert.ToInt64(header.TotalIonCurrent),
PrecursorMz = header.ParentIonMZ,
CollisionType = CollisionType.Other,
};
switch (header.CollisionMode)
{
case "cid":
summary.CollisionType = CollisionType.Cid;
break;
case "hcd":
summary.CollisionType = CollisionType.Hcd;
break;
case "etd":
summary.CollisionType = CollisionType.Etd;
break;
case "ecd":
summary.CollisionType = CollisionType.Ecd;
break;
case "hid":
summary.CollisionType = CollisionType.Hid;
break;
}
return summary;
}
private XRawFileIO GetReaderForGroup(int groupId)
{
if (!m_dataFiles.ContainsKey(groupId))
{
throw new Exception("The group-dataset ID provided was not found.");
}
// If we dont have a reader, then create one for this group
// next time, it will be available and we won't have to waste time
// opening the file.
if (!m_readers.ContainsKey(groupId))
{
var path = m_dataFiles[groupId];
var reader = new XRawFileIO();
m_readers.Add(groupId, reader);
var opened = reader.OpenRawFile(path);
if (!opened)
{
throw new IOException("Could not open the Thermo raw file " + m_dataFiles[groupId]);
}
}
var rawReader = m_readers[groupId];
return rawReader;
}
public RawFileReader()
{
_RawData = new XRawFileIO();
}
public RawFileReader(string inpFile)
{
_RawData = new XRawFileIO();
_RawData.OpenRawFile(inpFile);
}
private List <udtScansDataType> LoadScansFile(string strScansFilePath)
{
const string FILTERED_SCANS_SUFFIX = "_filtered_scans.csv";
var dctColumnInfo = new Dictionary <string, int>
{
{ "type", -1 },
{ "bpi", -1 },
{ "bpi_mz", -1 },
{ "tic", -1 },
{ "num_peaks", -1 },
{ "num_deisotoped", -1 }
};
var intColIndexScanOrFrameNum = -1;
var intColIndexScanOrFrameTime = -1;
var lstScanData = new List <udtScansDataType>();
var intRowNumber = 0;
var intColIndexScanInfo = -1;
if (!File.Exists(strScansFilePath) && strScansFilePath.ToLower().EndsWith(FILTERED_SCANS_SUFFIX))
{
strScansFilePath = strScansFilePath.Substring(0, strScansFilePath.Length - FILTERED_SCANS_SUFFIX.Length) + "_scans.csv";
}
if (!File.Exists(strScansFilePath))
{
OnWarningEvent("Warning: _scans.csv file is missing; will plot vs. scan number instead of vs. elution time");
return(lstScanData);
}
Console.WriteLine(" Reading the _scans.csv file");
using (var srIsosFile = new StreamReader(new FileStream(strScansFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
{
while (!srIsosFile.EndOfStream)
{
intRowNumber += 1;
var strLineIn = srIsosFile.ReadLine();
if (strLineIn == null)
{
continue;
}
var lstData = strLineIn.Split(',').ToList();
if (intRowNumber == 1)
{
// Parse the header row
ParseColumnHeaders(dctColumnInfo, lstData, "_scans.csv");
intColIndexScanOrFrameNum = GetScanOrFrameColIndex(lstData, "_scans.csv");
intColIndexScanOrFrameTime = lstData.IndexOf("frame_time");
if (intColIndexScanOrFrameTime < 0)
{
intColIndexScanOrFrameTime = lstData.IndexOf("scan_time");
}
// The info column will have data of the form "FTMS + p NSI Full ms [400.00-2000.00]" for Thermo datasets
// For .mzXML files, this fill will simply have an integer (and thus isn't useful)
// It may not be present in older _scancs.csv files and is thus optional
intColIndexScanInfo = lstData.IndexOf("info");
continue;
}
try
{
var udtScanData = new udtScansDataType();
int.TryParse(lstData[intColIndexScanOrFrameNum], out udtScanData.Scan);
float.TryParse(lstData[intColIndexScanOrFrameTime], out udtScanData.ElutionTime);
int.TryParse(lstData[dctColumnInfo["type"]], out udtScanData.MSLevel);
double.TryParse(lstData[dctColumnInfo["bpi"]], out udtScanData.BasePeakIntensity);
double.TryParse(lstData[dctColumnInfo["bpi_mz"]], out udtScanData.BasePeakMZ);
double.TryParse(lstData[dctColumnInfo["tic"]], out udtScanData.TotalIonCurrent);
int.TryParse(lstData[dctColumnInfo["num_peaks"]], out udtScanData.NumPeaks);
int.TryParse(lstData[dctColumnInfo["num_deisotoped"]], out udtScanData.NumDeisotoped);
if (intColIndexScanInfo > 0)
{
var infoText = lstData[intColIndexScanInfo];
// Only store infoText in .FilterText if infoText is not simply an integer
if (!int.TryParse(infoText, out _))
{
udtScanData.FilterText = infoText;
}
}
lstScanData.Add(udtScanData);
if (mSaveTICAndBPI)
{
mTICandBPIPlot.AddData(udtScanData.Scan, udtScanData.MSLevel, udtScanData.ElutionTime, udtScanData.BasePeakIntensity, udtScanData.TotalIonCurrent);
}
string scanTypeName;
if (string.IsNullOrWhiteSpace(udtScanData.FilterText))
{
udtScanData.FilterText = udtScanData.MSLevel > 1 ? "HMSn" : "HMS";
scanTypeName = udtScanData.FilterText;
}
else
{
scanTypeName = XRawFileIO.GetScanTypeNameFromFinniganScanFilterText(udtScanData.FilterText);
}
var objScanStatsEntry = new clsScanStatsEntry
{
ScanNumber = udtScanData.Scan,
ScanType = udtScanData.MSLevel,
ScanTypeName = scanTypeName,
ScanFilterText = XRawFileIO.MakeGenericFinniganScanFilter(udtScanData.FilterText),
ElutionTime = udtScanData.ElutionTime.ToString("0.0###"),
TotalIonIntensity = StringUtilities.ValueToString(udtScanData.TotalIonCurrent, 5),
BasePeakIntensity = StringUtilities.ValueToString(udtScanData.BasePeakIntensity, 5),
BasePeakMZ = udtScanData.BasePeakMZ.ToString("0.0###"),
BasePeakSignalToNoiseRatio = "0",
IonCount = udtScanData.NumDeisotoped,
IonCountRaw = udtScanData.NumPeaks,
ExtendedScanInfo =
{
CollisionMode = string.Empty,
ScanFilterText = udtScanData.FilterText
}
};
mDatasetStatsSummarizer.AddDatasetScan(objScanStatsEntry);
}
catch (Exception ex)
{
OnWarningEvent("Warning: Ignoring scan " + lstData[dctColumnInfo["scan_num"]] + " since data conversion error: " + ex.Message);
}
}
}
return(lstScanData);
}
private static void TestScanFilterParsing()
{
// Note: See also the unit test classes in the ThermoRawFileReaderUnitTests project
var filterList = new List <string>
{
"ITMS + c ESI Full ms [300.00-2000.00]",
"FTMS + p NSI Full ms [400.00-2000.00]",
"ITMS + p ESI d Z ms [579.00-589.00]",
"ITMS + c ESI d Full ms2 [email protected] [150.00-1180.00]",
"ITMS + c NSI d Full ms2 [email protected] [50.00-2000.00]",
"FTMS + c NSI d Full ms2 [email protected] [100.00-2000.00]",
"ITMS + c NSI d sa Full ms2 [email protected] [50.00-2000.00]",
"+ c d Full ms2 [email protected] [ 350.00-2000.00]",
"+ c d Full ms3 [email protected] [email protected] [ 350.00-2000.00]",
"ITMS + c NSI d Full ms10 [email protected]",
"+ p ms2 [email protected] [210.00-1200.00]",
"+ c NSI SRM ms2 [email protected] [507.259-507.261, 635-319-635.32]",
"+ c NSI SRM ms2 748.371 [701.368-701.370, 773.402-773.404, 887.484-887.486, 975.513-975.515]",
"+ p NSI Q1MS [179.652-184.582, 505.778-510.708, 994.968-999.898]",
"+ p NSI Q3MS [150.070-1500.000]",
"c NSI Full cnl 162.053 [300.000-1200.000]",
"- p NSI Full ms2 168.070 [300.000-1500.00]",
"+ c NSI Full ms2 1083.000 [300.000-1500.00]",
"- p NSI Full ms2 247.060 [300.000-1500.00]",
"- c NSI d Full ms2 921.597 [300.000-1500.00]",
"+ c NSI SRM ms2 965.958 [300.000-1500.00]",
"+ p NSI SRM ms2 1025.250 [300.000-1500.00]",
"+ c EI SRM ms2 247.000 [300.000-1500.00]",
"+ p NSI Full ms2 589.840 [300.070-1200.000]",
"+ p NSI ms [0.316-316.000]",
"ITMS + p NSI SIM ms",
"ITMS + c NSI d SIM ms",
"FTMS + p NSI SIM ms",
"FTMS + p NSI d SIM ms"
};
foreach (var filterItem in filterList)
{
var genericFilter = XRawFileIO.MakeGenericFinniganScanFilter(filterItem);
var scanType = XRawFileIO.GetScanTypeNameFromFinniganScanFilterText(filterItem);
Console.WriteLine(filterItem);
Console.WriteLine(" {0,-12} {1}", scanType, genericFilter);
var validMS1Scan = XRawFileIO.ValidateMSScan(filterItem, out var msLevel, out var simScan, out var mrmScanType, out var zoomScan);
if (validMS1Scan)
{
if (mrmScanType == MRMScanTypeConstants.NotMRM)
{
Console.Write(" MSLevel={0}", msLevel);
}
else
{
Console.Write(" MSLevel={0}, MRMScanType={1}", msLevel, mrmScanType);
}
if (simScan)
{
Console.Write(", SIM Scan");
}
if (zoomScan)
{
Console.Write(", Zoom Scan");
}
Console.WriteLine();
}
else
{
Console.WriteLine(" Not an MS1, SRM, MRM, or SIM scan");
}
Console.WriteLine();
}
Console.WriteLine();
}
private SortedSet <float> GetUniqueCvValues(XRawFileIO reader)
{
OnDebugEvent("Determining FAIMS CV values");
var lastStatus = DateTime.UtcNow;
var progressShown = false;
// Dictionary where keys are CV values and values are the filter text that scans with this CV value will have
var cvValues = new SortedSet <float>();
// Dictionary where keys are CV values and values are the number of scans with this value
// This is used when Options.Preview is true
var cvValueStats = new Dictionary <float, int>();
for (var scanNumber = reader.ScanStart; scanNumber <= reader.ScanEnd; scanNumber++)
{
if (DateTime.UtcNow.Subtract(lastStatus).TotalSeconds >= 3)
{
var percentComplete = scanNumber / (float)reader.ScanEnd * 100;
OnDebugEvent(string.Format(" ... {0:F0}% of scans examined", percentComplete));
lastStatus = DateTime.UtcNow;
progressShown = true;
}
var success = GetCvValue(reader, scanNumber, out var cvValue, out _, true);
if (!success)
{
continue;
}
if (cvValues.Contains(cvValue))
{
var scanCount = cvValueStats[cvValue];
cvValueStats[cvValue] = scanCount + 1;
if (QuickCVLookup && scanCount > 50)
{
// If all of the CV values have been found 50+ times, assume that we have found all of the CV values
// (since typically machines cycle through a CV list)
var minObservedCount = (from item in cvValueStats select item.Value).Min();
if (minObservedCount > 50)
{
// Ignore the remaining scans
break;
}
}
continue;
}
cvValues.Add(cvValue);
cvValueStats.Add(cvValue, 1);
}
if (progressShown)
{
OnDebugEvent(string.Format(" ... {0:F0}% of scans examined", 100));
}
return(cvValues);
}
