/// <summary>
/// Constructor
/// </summary>
/// <param name="thermoRawFilePath"></param>
/// <param name="loadTuneInfo"></param>
public XCaliburRun2(string thermoRawFilePath, bool loadTuneInfo = false)
{
Check.Require(File.Exists(thermoRawFilePath), "Run not initialized. File not found: " + thermoRawFilePath);
DatasetFileOrDirectoryPath = thermoRawFilePath;
var baseFilename = Path.GetFileName(DatasetFileOrDirectoryPath);
if (baseFilename == null)
{
throw new FileNotFoundException("Unable to determine the filename for " + thermoRawFilePath);
}
DatasetName = baseFilename.Substring(0, baseFilename.LastIndexOf('.'));
DatasetDirectoryPath = Path.GetDirectoryName(thermoRawFilePath);
var options = new ThermoReaderOptions
{
LoadMSMethodInfo = false,
LoadMSTuneInfo = loadTuneInfo
};
mRawFileReader = new XRawFileIO(DatasetFileOrDirectoryPath, options);
IsDataThresholded = true;
IsMsAbundanceReportedAsAverage = true;
MSFileType = Globals.MSFileType.Thermo_Raw;
ContainsMSMSData = true;
XYData = new XYData();
ParentScanList = new Dictionary <int, int>();
MinLCScan = 1;
MaxLCScan = GetNumMSScans();
}
private bool ProcessFile(string filePath, string outputDirectoryPath)
{
try
{
OnStatusEvent(string.Format("Processing {0}", filePath));
// Compute the SHA-1 hash of the .Raw file
var fileSha1 = HashRawFile(filePath);
// Disable loading the method
// Method loading doesn't work on Linux and this program doesn't need that information
var readerOptions = new ThermoReaderOptions
{
LoadMSMethodInfo = false
};
// Open up the raw file connection
var reader = new XRawFileIO(filePath, readerOptions);
// Get all unique CV values from scans
var cvValues = GetUniqueCvValues(reader);
var totalScanCount = reader.GetNumScans();
var scansProcessed = 0;
var lastProgress = DateTime.UtcNow;
// Reset tracking variables
ByteTracking.Reset(true);
// Now work for each unique CV value (# files we're going to need to split into)
// get all scans that have the CV that we're currently targeting
foreach (var cvValue in cvValues)
{
var baseName = Path.GetFileNameWithoutExtension(filePath);
var mzXmlPath = Path.Combine(outputDirectoryPath, baseName + "_" + cvValue + ".mzXML");
OnDebugEvent(string.Format("Creating file {0}", mzXmlPath));
var targetScans = FindAllTargetScans(cvValue, reader);
var parentScanToMS2Map = new Dictionary <int, List <int> >();
using (var writer = new StreamWriter(new FileStream(mzXmlPath, FileMode.Create, FileAccess.Write, FileShare.Read)))
{
WriteHeader(writer, filePath, reader, fileSha1, targetScans);
Ms1Scan currentMS1Scan = null;
// Write out our target scans
foreach (var scanNumber in targetScans)
{
if (DateTime.UtcNow.Subtract(lastProgress).TotalSeconds >= 3)
{
var percentComplete = scansProcessed / (double)totalScanCount * 100;
OnDebugEvent(string.Format("... processing: {0:F0}% complete", percentComplete));
lastProgress = DateTime.UtcNow;
}
var scan = new MsScan(scanNumber, reader);
if (scan.MsLevel == 1)
{
parentScanToMS2Map.Add(scan.ScanNumber, new List <int>());
if (currentMS1Scan == null)
{
// Start condition
currentMS1Scan = Ms1Scan.Create(scan);
}
else
{
// Write currentMS1Scan to file
var outString = currentMS1Scan.ToXML(this);
writer.WriteLine(outString);
currentMS1Scan = Ms1Scan.Create(scan);
}
}
else if (scan.MsLevel == 2)
{
if (currentMS1Scan != null)
{
parentScanToMS2Map[currentMS1Scan.ScanNumber].Add(scan.ScanNumber);
}
var ms2Scan = Ms2Scan.Create(scan);
ms2Scan.AddMs2ScanParameters(reader);
currentMS1Scan?.AddMs2Scan(ms2Scan);
}
scansProcessed++;
}
if (currentMS1Scan != null)
{
// Once we're out, we need to write out our last currentMS1Scan
writer.WriteLine(currentMS1Scan.ToXML(this));
}
// Finish off msRun
writer.WriteLine(" </msRun>");
writer.WriteLine(" <index name=\"scan\">");
// Add special entry to our indexOffset list for where the offsets start
var index = new Index(0, ByteTracking.ByteDepth + ByteTracking.Encoder.GetByteCount(" </msRun>") + 3);
ByteTracking.ScanOffsets.Add(index);
// Write all index offsets
for (var i = 0; i < ByteTracking.ScanOffsets.Count - 1; i++)
{
var offset = ByteTracking.ScanOffsets[i];
writer.WriteLine(" <offset id=\"" + offset.ScanNumber + "\">" + offset.ByteDepth + "</offset>");
}
writer.WriteLine(" </index>");
writer.WriteLine(" <indexOffset>" + ByteTracking.ScanOffsets.Last().ByteDepth + "</indexOffset>");
writer.Write(" <sha1>");
}
// Compute the SHA-1 hash of the file up to this point
var mzXmlHash = HashMzXML(mzXmlPath);
// Append the hash
using (var writer = new StreamWriter(mzXmlPath, true))
{
writer.Write(mzXmlHash);
writer.WriteLine("</sha1>");
writer.WriteLine("</mzXML>");
}
// Reset tracking variables, but do not restart scan numbering
ByteTracking.Reset(false);
}
OnDebugEvent(string.Format("... processing: {0:F0}% complete", 100));
return(true);
}
catch (Exception ex)
{
OnErrorEvent("Error in ProcessFile", ex);
return(false);
}
}
private static void TestGetAllScanEvents(string rawFilePath)
{
try
{
var rawFile = ResolveDataFile(rawFilePath);
if (rawFile == null)
{
return;
}
Console.WriteLine();
Console.WriteLine("Opening " + rawFile.FullName);
// Keys in this dictionary are event names
// Values are dictionaries tracking all of the values for each event (key is value, value is occurrence of that value)
var scanEventStats = new Dictionary <string, Dictionary <string, int> >();
var lastProgress = DateTime.UtcNow;
var scansRead = 0;
var scansReadSinceLastProgress = 0;
var options = new ThermoReaderOptions
{
LoadMSMethodInfo = mLoadMethods
};
using (var reader = new XRawFileIO(rawFile.FullName, options))
{
var scanCount = reader.GetNumScans();
for (var scanNum = 1; scanNum <= scanCount; scanNum++)
{
var success = reader.GetScanInfo(scanNum, out clsScanInfo scanInfo);
if (!success)
{
ShowWarning("GetScanInfo returned false for scan " + scanNum);
continue;
}
foreach (var eventItem in scanInfo.ScanEvents)
{
if (!scanEventStats.TryGetValue(eventItem.Key, out var valueStats))
{
valueStats = new Dictionary <string, int>();
scanEventStats.Add(eventItem.Key, valueStats);
}
if (valueStats.TryGetValue(eventItem.Value, out var valueCount))
{
valueStats[eventItem.Value] = valueCount + 1;
}
else
{
valueStats.Add(eventItem.Value, 1);
}
}
scansRead++;
scansReadSinceLastProgress++;
var elapsedSeconds = DateTime.UtcNow.Subtract(lastProgress).TotalSeconds;
if (!(elapsedSeconds > 3) || scansRead % 100 != 0)
{
continue;
}
lastProgress = DateTime.UtcNow;
var percentComplete = scansRead / (double)scanCount * 100;
var scansPerSecond = scansReadSinceLastProgress / elapsedSeconds;
Console.WriteLine("Reading scan events; {0:F1}% complete ({1} / {2} scans); {3:F0} scans/second",
percentComplete, scansRead, scanCount, scansPerSecond);
scansReadSinceLastProgress = 0;
}
}
Console.WriteLine();
Console.WriteLine("{0,-38} {1,-10} {2,-20} {3}", "Event name", "Values", "Most Common Value", "Occurrence count");
foreach (var eventInfo in scanEventStats)
{
var eventName = eventInfo.Key;
var valueCount = eventInfo.Value.Count;
var occurrenceCount = 0;
var mostCommonValue = "";
foreach (var item in eventInfo.Value)
{
if (item.Value <= occurrenceCount)
{
continue;
}
occurrenceCount = item.Value;
mostCommonValue = item.Key;
}
if (string.IsNullOrWhiteSpace(mostCommonValue))
{
Console.WriteLine("{0,-38} {1,-10} {2,-20} {3:N0}", eventName, valueCount, "whitespace", occurrenceCount);
}
else
{
Console.WriteLine("{0,-38} {1,-10} {2,-20} {3:N0}", eventName, valueCount, mostCommonValue, occurrenceCount);
}
}
Console.WriteLine();
}
catch (Exception ex)
{
ShowError("Error in TestGetAllScanEvents: " + ex.Message, ex);
}
}
private static void TestReader(string rawFilePath, bool centroid = false, bool testSumming = false, int scanStart = 0, int scanEnd = 0)
{
try
{
var rawFile = ResolveDataFile(rawFilePath);
if (rawFile == null)
{
return;
}
var options = new ThermoReaderOptions {
LoadMSMethodInfo = mLoadMethods
};
using (var reader = new XRawFileIO(rawFile.FullName, options, mTraceMode))
{
RegisterEvents(reader);
var numScans = reader.GetNumScans();
var collisionEnergyList = string.Empty;
ShowMethod(reader);
var scanStep = 1;
if (scanStart < 1)
{
scanStart = 1;
}
if (scanEnd < 1)
{
scanEnd = numScans;
scanStep = 21;
}
else
{
if (scanEnd < scanStart)
{
scanEnd = scanStart;
}
}
if (scanEnd > numScans)
{
scanEnd = numScans;
}
var msLevelStats = new Dictionary <int, int>();
Console.WriteLine();
Console.WriteLine("Reading data for scans {0} to {1}, step {2}", scanStart, scanEnd, scanStep);
for (var scanNum = scanStart; scanNum <= scanEnd; scanNum += scanStep)
{
if (scanNum > reader.ScanEnd)
{
ConsoleMsgUtils.ShowWarning("Exiting for loop since scan number {0} is greater than the max scan number, {1}", scanNum, reader.ScanEnd);
break;
}
if (mOnlyLoadMSLevelInfo)
{
var msLevel = reader.GetMSLevel(scanNum);
if (msLevelStats.TryGetValue(msLevel, out var msLevelObsCount))
{
msLevelStats[msLevel] = msLevelObsCount + 1;
}
else
{
msLevelStats.Add(msLevel, 1);
}
if (mScanInfoInterval <= 0 || scanNum % mScanInfoInterval == 0)
{
Console.WriteLine("Scan " + scanNum);
}
continue;
}
var success = reader.GetScanInfo(scanNum, out clsScanInfo scanInfo);
if (!success)
{
continue;
}
if (mScanInfoInterval <= 0 || scanNum % mScanInfoInterval == 0)
{
Console.WriteLine("Scan " + scanNum + " at " + scanInfo.RetentionTime.ToString("0.00") + " minutes: " + scanInfo.FilterText);
}
if (mLoadCollisionEnergies)
{
var collisionEnergies = reader.GetCollisionEnergy(scanNum);
if (collisionEnergies.Count == 0)
{
collisionEnergyList = string.Empty;
}
else if (collisionEnergies.Count >= 1)
{
collisionEnergyList = collisionEnergies[0].ToString("0.0");
if (collisionEnergies.Count > 1)
{
for (var index = 1; index <= collisionEnergies.Count - 1; index++)
{
collisionEnergyList += ", " + collisionEnergies[index].ToString("0.0");
}
}
}
if (string.IsNullOrEmpty(collisionEnergyList))
{
Console.WriteLine();
}
else
{
Console.WriteLine("; CE " + collisionEnergyList);
}
}
if (mGetScanEvents)
{
if (scanInfo.TryGetScanEvent("Monoisotopic M/Z:", out var monoMZ))
{
Console.WriteLine("Monoisotopic M/Z: " + monoMZ);
}
if (scanInfo.TryGetScanEvent("Charge State", out var chargeState, true))
{
Console.WriteLine("Charge State: " + chargeState);
}
if (scanInfo.TryGetScanEvent("MS2 Isolation Width", out var isolationWidth, true))
{
Console.WriteLine("MS2 Isolation Width: " + isolationWidth);
}
}
if (!mLoadScanData || (scanNum % 50 != 0 && scanEnd - scanStart > 50))
{
continue;
}
// Get the data for scan scanNum
Console.WriteLine();
Console.WriteLine("Spectrum for scan " + scanNum);
reader.GetScanData(scanNum, out var mzList, out var intensityList, 0, centroid);
var mzDisplayStepSize = 50;
if (centroid)
{
mzDisplayStepSize = 1;
}
for (var i = 0; i <= mzList.Length - 1; i += mzDisplayStepSize)
{
Console.WriteLine(" " + mzList[i].ToString("0.000") + " mz " + intensityList[i].ToString("0"));
}
Console.WriteLine();
const int scansToSum = 15;
if (scanNum + scansToSum < numScans && testSumming)
{
// Get the data for scan scanNum through scanNum + 15
#pragma warning disable 618
reader.GetScanDataSumScans(scanNum, scanNum + scansToSum, out var massIntensityPairs, 0, centroid);
#pragma warning restore 618
Console.WriteLine("Summed spectrum, scans " + scanNum + " through " + (scanNum + scansToSum));
for (var i = 0; i <= massIntensityPairs.GetLength(1) - 1; i += 50)
{
Console.WriteLine(" " + massIntensityPairs[0, i].ToString("0.000") + " mz " +
massIntensityPairs[1, i].ToString("0"));
}
Console.WriteLine();
}
if (!scanInfo.IsFTMS)
{
continue;
}
var dataCount = reader.GetScanLabelData(scanNum, out var ftLabelData);
Console.WriteLine();
Console.WriteLine("{0,12}{1,12}{2,12}{3,12}{4,12}{5,12}", "Mass", "Intensity", "Resolution", "Baseline", "Noise", "Charge");
for (var i = 0; i <= dataCount - 1; i += 50)
{
Console.WriteLine("{0,12:F3}{1,12:0}{2,12:0}{3,12:F1}{4,12:0}{5,12:0}",
ftLabelData[i].Mass,
ftLabelData[i].Intensity,
ftLabelData[i].Resolution,
ftLabelData[i].Baseline,
ftLabelData[i].Noise,
ftLabelData[i].Charge);
}
dataCount = reader.GetScanPrecisionData(scanNum, out var ftPrecisionData);
Console.WriteLine();
Console.WriteLine("{0,12}{1,12}{2,12}{3,12}{4,12}", "Mass", "Intensity", "AccuracyMMU", "AccuracyPPM", "Resolution");
for (var i = 0; i <= dataCount - 1; i += 50)
{
Console.WriteLine("{0,12:F3}{1,12:0}{2,12:F3}{3,12:F3}{4,12:0}",
ftPrecisionData[i].Mass,
ftPrecisionData[i].Intensity, ftPrecisionData[i].AccuracyMMU,
ftPrecisionData[i].AccuracyPPM,
ftPrecisionData[i].Resolution);
}
}
if (mOnlyLoadMSLevelInfo)
{
Console.WriteLine();
Console.WriteLine("{0,-10} {1}", "MSLevel", "Scans");
foreach (var item in msLevelStats)
{
Console.WriteLine("{0, -10} {1}", item.Key, item.Value);
}
}
}
}
catch (Exception ex)
{
ShowError("Error in TestReader: " + ex.Message, ex);
}
}
/// <summary>
/// Process the dataset
/// </summary>
/// <param name="dataFilePath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error or if the file has no scans</returns>
/// <remarks></remarks>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
ResetResults();
var dataFilePathLocal = string.Empty;
// Obtain the full path to the file
var rawFile = new FileInfo(dataFilePath);
if (!rawFile.Exists)
{
OnErrorEvent(".Raw file not found: " + dataFilePath);
return(false);
}
// Future, optional: Determine the DatasetID
// Unfortunately, this is not present in metadata.txt
// datasetID = LookupDatasetID(datasetName)
var datasetID = DatasetID;
// Record the file size and Dataset ID
datasetFileInfo.FileSystemCreationTime = rawFile.CreationTime;
datasetFileInfo.FileSystemModificationTime = rawFile.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = datasetID;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(rawFile.Name);
datasetFileInfo.FileExtension = rawFile.Extension;
datasetFileInfo.FileSizeBytes = rawFile.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
var deleteLocalFile = false;
var readError = false;
// Note: as of June 2018 this only works if you disable "Prefer 32-bit" when compiling as AnyCPU
// Use XRaw to read the .Raw file
// If reading from a SAMBA-mounted network share, and if the current user has
// Read privileges but not Read&Execute privileges, we will need to copy the file locally
var readerOptions = new ThermoReaderOptions
{
LoadMSMethodInfo = true,
LoadMSTuneInfo = true
};
var xcaliburAccessor = new XRawFileIO(readerOptions);
RegisterEvents(xcaliburAccessor);
// Open a handle to the data file
if (!xcaliburAccessor.OpenRawFile(rawFile.FullName))
{
// File open failed
OnErrorEvent("Call to .OpenRawFile failed for: " + rawFile.FullName);
ErrorCode = iMSFileInfoScanner.eMSFileScannerErrorCodes.ThermoRawFileReaderError;
readError = true;
if (!string.Equals(clsMSFileInfoScanner.GetAppDirectoryPath().Substring(0, 2), rawFile.FullName.Substring(0, 2), StringComparison.InvariantCultureIgnoreCase))
{
if (mCopyFileLocalOnReadError)
{
// Copy the file locally and try again
try
{
dataFilePathLocal = Path.Combine(clsMSFileInfoScanner.GetAppDirectoryPath(), Path.GetFileName(dataFilePath));
if (!string.Equals(dataFilePathLocal, dataFilePath, StringComparison.InvariantCultureIgnoreCase))
{
OnDebugEvent("Copying file " + Path.GetFileName(dataFilePath) + " to the working directory");
File.Copy(dataFilePath, dataFilePathLocal, true);
dataFilePath = string.Copy(dataFilePathLocal);
deleteLocalFile = true;
// Update rawFile then try to re-open
rawFile = new FileInfo(dataFilePath);
if (!xcaliburAccessor.OpenRawFile(rawFile.FullName))
{
// File open failed
OnErrorEvent("Call to .OpenRawFile failed for: " + rawFile.FullName);
ErrorCode = iMSFileInfoScanner.eMSFileScannerErrorCodes.ThermoRawFileReaderError;
readError = true;
}
else
{
readError = false;
}
}
}
catch (Exception)
{
readError = true;
}
}
}
}
if (!readError)
{
// Read the file info
try
{
datasetFileInfo.AcqTimeStart = xcaliburAccessor.FileInfo.CreationDate;
}
catch (Exception)
{
// Read error
readError = true;
}
if (!readError)
{
try
{
datasetFileInfo.ScanCount = xcaliburAccessor.GetNumScans();
if (datasetFileInfo.ScanCount > 0)
{
// Look up the end scan time then compute .AcqTimeEnd
var scanEnd = xcaliburAccessor.FileInfo.ScanEnd;
xcaliburAccessor.GetScanInfo(scanEnd, out clsScanInfo scanInfo);
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(scanInfo.RetentionTime);
}
else
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
}
catch (Exception)
{
// Error; use default values
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
datasetFileInfo.ScanCount = 0;
}
if (mSaveTICAndBPI || mCreateDatasetInfoFile || mCreateScanStatsFile ||
mSaveLCMS2DPlots || mCheckCentroidingStatus || MS2MzMin > 0)
{
// Load data from each scan
// This is used to create the TIC and BPI plot, the 2D LC/MS plot, and/or to create the Dataset Info File
LoadScanDetails(xcaliburAccessor);
}
if (mComputeOverallQualityScores)
{
// Note that this call will also create the TICs and BPIs
ComputeQualityScores(xcaliburAccessor, datasetFileInfo);
}
if (MS2MzMin > 0 && datasetFileInfo.ScanCount > 0)
{
// Verify that all of the MS2 spectra have m/z values below the required minimum
// Useful for validating that reporter ions can be detected
ValidateMS2MzMin();
}
}
}
mDatasetStatsSummarizer.SampleInfo.SampleName = xcaliburAccessor.FileInfo.SampleName;
mDatasetStatsSummarizer.SampleInfo.Comment1 = xcaliburAccessor.FileInfo.Comment1;
mDatasetStatsSummarizer.SampleInfo.Comment2 = xcaliburAccessor.FileInfo.Comment2;
// Add the devices
var deviceFilterList = new SortedSet <Device> {
Device.MS,
Device.MSAnalog
};
// First add the MS devices
AddThermoDevices(xcaliburAccessor, datasetFileInfo, deviceFilterList, new SortedSet <Device>());
// Now add any non-mass spec devices
AddThermoDevices(xcaliburAccessor, datasetFileInfo, new SortedSet <Device>(), deviceFilterList);
if (mSaveTICAndBPI && datasetFileInfo.DeviceList.Count > 0)
{
mInstrumentSpecificPlots.Clear();
foreach (var device in datasetFileInfo.DeviceList)
{
if (device.DeviceType == Device.MS || device.DeviceType == Device.MSAnalog)
{
continue;
}
var chromatogramData = xcaliburAccessor.GetChromatogramData(device.DeviceType, device.DeviceNumber);
if (chromatogramData.Count == 0)
{
continue;
}
var devicePlot = AddInstrumentSpecificPlot(device.DeviceDescription);
devicePlot.TICXAxisLabel = string.IsNullOrWhiteSpace(device.AxisLabelX) ? "Scan number" : device.AxisLabelX;
devicePlot.TICYAxisLabel = device.YAxisLabelWithUnits;
devicePlot.TICYAxisExponentialNotation = false;
devicePlot.DeviceType = device.DeviceType;
devicePlot.TICPlotAbbrev = string.Format("{0}{1}", device.DeviceType.ToString(), device.DeviceNumber);
devicePlot.TICAutoMinMaxY = true;
devicePlot.RemoveZeroesFromEnds = false;
float acqLengthMinutes;
if (datasetFileInfo.AcqTimeEnd > datasetFileInfo.AcqTimeStart)
{
acqLengthMinutes = (float)datasetFileInfo.AcqTimeEnd.Subtract(datasetFileInfo.AcqTimeStart).TotalMinutes;
}
else
{
acqLengthMinutes = chromatogramData.Count;
}
var dataCount = chromatogramData.Count;
foreach (var dataPoint in chromatogramData)
{
var scanNumber = dataPoint.Key;
float scanTimeMinutes;
if (acqLengthMinutes > 0)
{
scanTimeMinutes = scanNumber / (float)dataCount * acqLengthMinutes;
}
else
{
scanTimeMinutes = scanNumber;
}
devicePlot.AddDataTICOnly(scanNumber, 1, scanTimeMinutes, dataPoint.Value);
}
}
}
if (!string.IsNullOrEmpty(xcaliburAccessor.FileInfo.SampleComment))
{
if (string.IsNullOrEmpty(mDatasetStatsSummarizer.SampleInfo.Comment1))
{
mDatasetStatsSummarizer.SampleInfo.Comment1 = xcaliburAccessor.FileInfo.SampleComment;
}
else
{
if (string.IsNullOrEmpty(mDatasetStatsSummarizer.SampleInfo.Comment2))
{
mDatasetStatsSummarizer.SampleInfo.Comment2 = xcaliburAccessor.FileInfo.SampleComment;
}
else
{
// Append the sample comment to comment 2
mDatasetStatsSummarizer.SampleInfo.Comment2 += "; " + xcaliburAccessor.FileInfo.SampleComment;
}
}
}
// Close the handle to the data file
xcaliburAccessor.CloseRawFile();
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
// This will also compute the SHA-1 hash of the .Raw file and add it to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetFileStats(rawFile, datasetID);
// Copy over the updated file time info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetStatsSummarizerUsingDatasetFileInfo(datasetFileInfo);
// Delete the local copy of the data file
if (deleteLocalFile)
{
try
{
File.Delete(dataFilePathLocal);
}
catch (Exception)
{
// Deletion failed
OnErrorEvent("Deletion failed for: " + Path.GetFileName(dataFilePathLocal));
}
}
PostProcessTasks();
return(!readError);
}
/// <summary>
/// Looks for the reporter ion peaks using FindReporterIonsWork
/// </summary>
/// <param name="scanList"></param>
/// <param name="spectraCache"></param>
/// <param name="inputFilePathFull">Full path to the input file</param>
/// <param name="outputDirectoryPath"></param>
/// <returns></returns>
/// <remarks></remarks>
public bool FindReporterIons(
clsScanList scanList,
clsSpectraCache spectraCache,
string inputFilePathFull,
string outputDirectoryPath)
{
const char TAB_DELIMITER = '\t';
var outputFilePath = "??";
try
{
// Use Xraw to read the .Raw files
var readerOptions = new ThermoReaderOptions()
{
LoadMSMethodInfo = false,
LoadMSTuneInfo = false
};
var rawFileReader = new XRawFileIO(readerOptions);
RegisterEvents(rawFileReader);
var includeFtmsColumns = false;
if (inputFilePathFull.ToUpper().EndsWith(DataInput.clsDataImport.THERMO_RAW_FILE_EXTENSION.ToUpper()))
{
// Processing a thermo .Raw file
// Check whether any of the frag scans has IsFTMS true
for (var masterOrderIndex = 0; masterOrderIndex < scanList.MasterScanOrderCount; masterOrderIndex++)
{
var scanPointer = scanList.MasterScanOrder[masterOrderIndex].ScanIndexPointer;
if (scanList.MasterScanOrder[masterOrderIndex].ScanType == clsScanList.eScanTypeConstants.SurveyScan)
{
// Skip survey scans
continue;
}
if (scanList.FragScans[scanPointer].IsFTMS)
{
includeFtmsColumns = true;
break;
}
}
if (includeFtmsColumns)
{
rawFileReader.OpenRawFile(inputFilePathFull);
}
}
if (mOptions.ReporterIons.ReporterIonList.Count == 0)
{
// No reporter ions defined; default to ITraq
mOptions.ReporterIons.SetReporterIonMassMode(clsReporterIons.eReporterIonMassModeConstants.ITraqFourMZ);
}
// Populate array reporterIons, which we will sort by m/z
var reporterIons = new clsReporterIonInfo[mOptions.ReporterIons.ReporterIonList.Count];
var reporterIonIndex = 0;
foreach (var reporterIon in mOptions.ReporterIons.ReporterIonList)
{
reporterIons[reporterIonIndex] = reporterIon;
reporterIonIndex += 1;
}
Array.Sort(reporterIons, new clsReportIonInfoComparer());
outputFilePath = clsDataOutput.ConstructOutputFilePath(
Path.GetFileName(inputFilePathFull),
outputDirectoryPath,
clsDataOutput.eOutputFileTypeConstants.ReporterIonsFile);
using (var writer = new StreamWriter(outputFilePath))
{
// Write the file headers
var reporterIonMZsUnique = new SortedSet <string>();
var headerColumns = new List <string>(7 + reporterIons.Length + 1)
{
"Dataset",
"ScanNumber",
"Collision Mode",
"ParentIonMZ",
"BasePeakIntensity",
"BasePeakMZ",
"ReporterIonIntensityMax"
};
var obsMZHeaders = new List <string>(reporterIons.Length);
var uncorrectedIntensityHeaders = new List <string>(reporterIons.Length);
var ftmsSignalToNoise = new List <string>(reporterIons.Length);
var ftmsResolution = new List <string>(reporterIons.Length);
//var ftmsLabelDataMz = new List<string>(reporterIons.Length);
var saveUncorrectedIntensities =
mOptions.ReporterIons.ReporterIonApplyAbundanceCorrection && mOptions.ReporterIons.ReporterIonSaveUncorrectedIntensities;
var dataAggregation = new clsDataAggregation();
RegisterEvents(dataAggregation);
foreach (var reporterIon in reporterIons)
{
if (!reporterIon.ContaminantIon || saveUncorrectedIntensities)
{
// Construct the reporter ion intensity header
// We skip contaminant ions, unless saveUncorrectedIntensities is True, then we include them
string mzValue;
if (mOptions.ReporterIons.ReporterIonMassMode == clsReporterIons.eReporterIonMassModeConstants.TMTTenMZ ||
mOptions.ReporterIons.ReporterIonMassMode == clsReporterIons.eReporterIonMassModeConstants.TMTElevenMZ ||
mOptions.ReporterIons.ReporterIonMassMode == clsReporterIons.eReporterIonMassModeConstants.TMTSixteenMZ)
{
mzValue = reporterIon.MZ.ToString("#0.000");
}
else
{
mzValue = ((int)Math.Round(reporterIon.MZ, 0)).ToString();
}
if (reporterIonMZsUnique.Contains(mzValue))
{
// Uniquify the m/z value
mzValue += "_" + reporterIonIndex;
}
try
{
reporterIonMZsUnique.Add(mzValue);
}
catch (Exception ex)
{
// Error updating the SortedSet;
// this shouldn't happen based on the .ContainsKey test above
}
// Append the reporter ion intensity title to the headers
headerColumns.Add("Ion_" + mzValue);
// This string will only be included in the header line if mOptions.ReporterIons.ReporterIonSaveObservedMasses is true
obsMZHeaders.Add("Ion_" + mzValue + "_ObsMZ");
// This string will be included in the header line if saveUncorrectedIntensities is true
uncorrectedIntensityHeaders.Add("Ion_" + mzValue + "_OriginalIntensity");
// This string will be included in the header line if includeFtmsColumns is true
ftmsSignalToNoise.Add("Ion_" + mzValue + "_SignalToNoise");
ftmsResolution.Add("Ion_" + mzValue + "_Resolution");
// Uncomment to include the label data m/z value in the _ReporterIons.txt file
// This string will only be included in the header line if mOptions.ReporterIons.ReporterIonSaveObservedMasses is true
//ftmsLabelDataMz.Add("Ion_" + mzValue + "_LabelDataMZ");
}
}
headerColumns.Add("Weighted Avg Pct Intensity Correction");
if (mOptions.ReporterIons.ReporterIonSaveObservedMasses)
{
headerColumns.AddRange(obsMZHeaders);
}
if (saveUncorrectedIntensities)
{
headerColumns.AddRange(uncorrectedIntensityHeaders);
}
if (includeFtmsColumns)
{
headerColumns.AddRange(ftmsSignalToNoise);
headerColumns.AddRange(ftmsResolution);
// Uncomment to include the label data m/z value in the _ReporterIons.txt file
// If mOptions.ReporterIons.ReporterIonSaveObservedMasses Then
// headerColumns.AddRange(ftmsLabelDataMz)
// End If
}
// Write the headers to the output file, separated by tabs
writer.WriteLine(string.Join(TAB_DELIMITER.ToString(), headerColumns));
UpdateProgress(0, "Searching for reporter ions");
for (var masterOrderIndex = 0; masterOrderIndex < scanList.MasterScanOrderCount; masterOrderIndex++)
{
var scanPointer = scanList.MasterScanOrder[masterOrderIndex].ScanIndexPointer;
if (scanList.MasterScanOrder[masterOrderIndex].ScanType == clsScanList.eScanTypeConstants.SurveyScan)
{
// Skip Survey Scans
continue;
}
FindReporterIonsWork(
rawFileReader,
dataAggregation,
includeFtmsColumns,
mOptions.SICOptions,
scanList,
spectraCache,
scanList.FragScans[scanPointer],
writer,
reporterIons,
TAB_DELIMITER,
saveUncorrectedIntensities,
mOptions.ReporterIons.ReporterIonSaveObservedMasses);
if (scanList.MasterScanOrderCount > 1)
{
UpdateProgress((short)(masterOrderIndex / (double)(scanList.MasterScanOrderCount - 1) * 100));
}
else
{
UpdateProgress(0);
}
UpdateCacheStats(spectraCache);
if (mOptions.AbortProcessing)
{
break;
}
}
}
if (includeFtmsColumns)
{
// Close the handle to the data file
rawFileReader.CloseRawFile();
}
return(true);
}
catch (Exception ex)
{
ReportError("Error writing the reporter ions to: " + outputFilePath, ex, clsMASIC.eMasicErrorCodes.OutputFileWriteError);
return(false);
}
}
/// <summary>
/// Read scan data and ions from a Thermo .raw file
/// </summary>
/// <param name="filePath"></param>
/// <param name="scanList"></param>
/// <param name="spectraCache"></param>
/// <param name="dataOutputHandler"></param>
/// <param name="keepRawSpectra"></param>
/// <param name="keepMSMSSpectra"></param>
/// <returns>True if Success, False if failure</returns>
/// <remarks>Assumes filePath exists</remarks>
public bool ExtractScanInfoFromXcaliburDataFile(
string filePath,
clsScanList scanList,
clsSpectraCache spectraCache,
clsDataOutput dataOutputHandler,
bool keepRawSpectra,
bool keepMSMSSpectra)
{
// Use XrawFileIO to read the .Raw files (it uses ThermoFisher.CommonCore)
var readerOptions = new ThermoReaderOptions
{
LoadMSMethodInfo = mOptions.WriteMSMethodFile,
LoadMSTuneInfo = mOptions.WriteMSTuneFile
};
var xcaliburAccessor = new XRawFileIO(readerOptions)
{
ScanInfoCacheMaxSize = 0 // Don't cache scanInfo objects
};
RegisterEvents(xcaliburAccessor);
mBpiUpdateCount = 0;
// Assume success for now
var success = true;
try
{
Console.Write("Reading Thermo .raw file ");
ReportMessage("Reading Thermo .raw file");
UpdateProgress(0, "Opening data file:" + Environment.NewLine + Path.GetFileName(filePath));
// Obtain the full path to the file
var rawFileInfo = new FileInfo(filePath);
var inputFileFullPath = rawFileInfo.FullName;
// Open a handle to the data file
if (!xcaliburAccessor.OpenRawFile(inputFileFullPath))
{
ReportError("Error opening input data file: " + inputFileFullPath + " (xcaliburAccessor.OpenRawFile returned False)");
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError);
return(false);
}
var datasetID = mOptions.SICOptions.DatasetID;
success = UpdateDatasetFileStats(rawFileInfo, datasetID, xcaliburAccessor);
var metadataWriter = new clsThermoMetadataWriter();
RegisterEvents(metadataWriter);
if (mOptions.WriteMSMethodFile)
{
metadataWriter.SaveMSMethodFile(xcaliburAccessor, dataOutputHandler);
}
if (mOptions.WriteMSTuneFile)
{
metadataWriter.SaveMSTuneFile(xcaliburAccessor, dataOutputHandler);
}
var scanCount = xcaliburAccessor.GetNumScans();
if (scanCount <= 0)
{
// No scans found
ReportError("No scans found in the input file: " + filePath);
SetLocalErrorCode(clsMASIC.eMasicErrorCodes.InputFileAccessError);
return(false);
}
var scanStart = xcaliburAccessor.ScanStart;
var scanEnd = xcaliburAccessor.ScanEnd;
InitOptions(scanList, keepRawSpectra, keepMSMSSpectra);
UpdateProgress(string.Format("Reading Xcalibur data ({0:N0} scans){1}", scanCount, Environment.NewLine + Path.GetFileName(filePath)));
ReportMessage(string.Format("Reading Xcalibur data; Total scan count: {0:N0}", scanCount));
var scanCountToRead = scanEnd - scanStart + 1;
var scansEst = mOptions.SICOptions.ScanRangeCount;
if (scansEst <= 0)
{
scansEst = scanCountToRead;
}
scanList.ReserveListCapacity(scansEst);
mScanTracking.ReserveListCapacity(scansEst);
spectraCache.SpectrumCount = Math.Max(spectraCache.SpectrumCount, scansEst);
for (var scanNumber = scanStart; scanNumber <= scanEnd; scanNumber++)
{
if (!mScanTracking.CheckScanInRange(scanNumber, mOptions.SICOptions))
{
mScansOutOfRange += 1;
continue;
}
success = xcaliburAccessor.GetScanInfo(scanNumber, out ThermoRawFileReader.clsScanInfo thermoScanInfo);
if (!success)
{
// GetScanInfo returned false
ReportWarning("xcaliburAccessor.GetScanInfo returned false for scan " + scanNumber.ToString() + "; aborting read");
break;
}
var percentComplete = scanList.MasterScanOrderCount / (double)(scanCountToRead) * 100;
var extractSuccess = ExtractScanInfoCheckRange(xcaliburAccessor, thermoScanInfo, scanList, spectraCache, dataOutputHandler, percentComplete);
if (!extractSuccess)
{
break;
}
}
Console.WriteLine();
scanList.SetListCapacityToCount();
mScanTracking.SetListCapacityToCount();
// Shrink the memory usage of the scanList arrays
success = FinalizeScanList(scanList, rawFileInfo);
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
ReportError("Error in ExtractScanInfoFromXcaliburDataFile", ex, clsMASIC.eMasicErrorCodes.InputFileDataReadError);
}
// Close the handle to the data file
xcaliburAccessor.CloseRawFile();
return(success);
}
