/// <summary>
/// This function is used to determine one or more overall quality scores
/// </summary>
/// <param name="objXcaliburAccessor"></param>
/// <param name="datasetFileInfo"></param>
/// <remarks></remarks>
private void ComputeQualityScores(XRawFileIO objXcaliburAccessor, clsDatasetFileInfo datasetFileInfo)
{
float sngOverallScore;
double dblOverallAvgIntensitySum = 0;
var intOverallAvgCount = 0;
if (mLCMS2DPlot.ScanCountCached > 0)
{
// Obtain the overall average intensity value using the data cached in mLCMS2DPlot
// This avoids having to reload all of the data using objXcaliburAccessor
const int intMSLevelFilter = 1;
sngOverallScore = mLCMS2DPlot.ComputeAverageIntensityAllScans(intMSLevelFilter);
}
else
{
var intScanCount = objXcaliburAccessor.GetNumScans();
GetStartAndEndScans(intScanCount, out var intScanStart, out var intScanEnd);
for (var intScanNumber = intScanStart; intScanNumber <= intScanEnd; intScanNumber++)
{
// This function returns the number of points in dblMassIntensityPairs()
var intReturnCode = objXcaliburAccessor.GetScanData2D(intScanNumber, out var dblMassIntensityPairs);
if (intReturnCode <= 0)
{
continue;
}
if ((dblMassIntensityPairs == null) || dblMassIntensityPairs.GetLength(1) <= 0)
{
continue;
}
// Keep track of the quality scores and then store one or more overall quality scores in datasetFileInfo.OverallQualityScore
// For now, this just computes the average intensity for each scan and then computes and overall average intensity value
double dblIntensitySum = 0;
for (var intIonIndex = 0; intIonIndex <= dblMassIntensityPairs.GetUpperBound(1); intIonIndex++)
{
dblIntensitySum += dblMassIntensityPairs[1, intIonIndex];
}
dblOverallAvgIntensitySum += dblIntensitySum / dblMassIntensityPairs.GetLength(1);
intOverallAvgCount += 1;
}
if (intOverallAvgCount > 0)
{
sngOverallScore = (float)(dblOverallAvgIntensitySum / intOverallAvgCount);
}
else
{
sngOverallScore = 0;
}
}
datasetFileInfo.OverallQualityScore = sngOverallScore;
}
/// <summary>
/// Reads the Contents.xml file to look for the AcquiredTime entry
/// </summary>
/// <param name="strFolderPath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if the file exists and the AcquiredTime entry was successfully parsed; otherwise false</returns>
/// <remarks></remarks>
private bool ProcessContentsXMLFile(string strFolderPath, clsDatasetFileInfo datasetFileInfo)
{
var blnSuccess = false;
try
{
// Open the Contents.xml file
var strFilePath = Path.Combine(strFolderPath, AGILENT_XML_CONTENTS_FILE);
using (var srReader = new System.Xml.XmlTextReader(new FileStream(strFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
while (!srReader.EOF)
{
srReader.Read();
switch (srReader.NodeType)
{
case System.Xml.XmlNodeType.Element:
if (srReader.Name == "AcquiredTime")
{
try
{
var dtAcquisitionStartTime = srReader.ReadElementContentAsDateTime();
// Convert from Universal time to Local time
var dtAcquisitionTime = dtAcquisitionStartTime.ToLocalTime();
// There have been some cases where the acquisition start time is several years before the file modification time,
// for example XG_A83CapiHSSWash1.d where the time in the Contents.xml file is 3/20/2005 while the file modification time is 2010
// Thus, we use a sanity check of a maximum run time of 24 hours
if (datasetFileInfo.AcqTimeEnd.Subtract(dtAcquisitionTime).TotalDays < 1)
{
datasetFileInfo.AcqTimeStart = dtAcquisitionStartTime.ToLocalTime();
blnSuccess = true;
}
}
catch (Exception)
{
// Ignore errors here
}
}
break;
}
}
}
}
catch (Exception ex)
{
// Exception reading file
OnErrorEvent("Exception reading " + AGILENT_XML_CONTENTS_FILE + ": " + ex.Message, ex);
blnSuccess = false;
}
return(blnSuccess);
}
private bool ParseGCIniFile(string strFolderPath, clsDatasetFileInfo datasetFileInfo)
{
double dblTotalRuntime = 0;
var blnSuccess = false;
try {
// Open the GC.ini file
var strFilePath = Path.Combine(strFolderPath, AGILENT_GC_INI_FILE);
if (!File.Exists(strFilePath))
{
return(false);
}
using (var srInFile = new StreamReader(strFilePath)) {
while (!srInFile.EndOfStream)
{
var strLineIn = srInFile.ReadLine();
if (string.IsNullOrWhiteSpace(strLineIn))
{
continue;
}
if (!strLineIn.StartsWith("gc.runlength"))
{
continue;
}
// Runtime is the value after the equals sign
var strSplitLine = strLineIn.Split('=');
if (strSplitLine.Length <= 1)
{
continue;
}
if (double.TryParse(strSplitLine[1], out dblTotalRuntime))
{
blnSuccess = true;
}
}
}
} catch (Exception ex) {
// Exception reading file
OnErrorEvent("Exception reading " + AGILENT_GC_INI_FILE + ": " + ex.Message, ex);
blnSuccess = false;
}
if (blnSuccess)
{
// Update the acquisition start time
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd.AddMinutes(-dblTotalRuntime);
}
return(blnSuccess);
}
/// <summary>
/// Process the DeconTools results
/// </summary>
/// <param name="strDataFilePath">Isos file path</param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error</returns>
/// <remarks>Will also read the _scans.csv file if present (to determine ElutionTime and MSLevel</remarks>
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
var fiIsosFile = new FileInfo(strDataFilePath);
if (!fiIsosFile.Exists)
{
OnErrorEvent("_isos.csv file not found: " + strDataFilePath);
return(false);
}
var intDatasetID = DatasetID;
// Record the file size and Dataset ID
datasetFileInfo.FileSystemCreationTime = fiIsosFile.CreationTime;
datasetFileInfo.FileSystemModificationTime = fiIsosFile.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = intDatasetID;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(fiIsosFile.Name);
datasetFileInfo.FileExtension = fiIsosFile.Extension;
datasetFileInfo.FileSizeBytes = fiIsosFile.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
if (mSaveTICAndBPI || mCreateDatasetInfoFile || mCreateScanStatsFile || mSaveLCMS2DPlots)
{
// Load data from each scan
// This is used to create the TIC and BPI plot, the 2D LC/MS plot, and/or to create the Dataset Info File
LoadData(fiIsosFile, datasetFileInfo);
}
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
UpdateDatasetFileStats(fiIsosFile, intDatasetID);
// Copy over the updated filetime info from datasetFileInfo to mDatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.FileSystemCreationTime = datasetFileInfo.FileSystemCreationTime;
mDatasetStatsSummarizer.DatasetFileInfo.FileSystemModificationTime = datasetFileInfo.FileSystemModificationTime;
mDatasetStatsSummarizer.DatasetFileInfo.DatasetID = datasetFileInfo.DatasetID;
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
return(true);
}
private bool ParseICRFolder(DirectoryInfo icrFolderInfo, clsDatasetFileInfo datasetFileInfo)
{
// Look for and open the .Pek file in ioFolderInfo
// Count the number of PEK_FILE_FILENAME_LINE lines
var intFileListCount = 0;
var blnSuccess = false;
foreach (var pekFile in icrFolderInfo.GetFiles("*.pek"))
{
try
{
// Try to open the PEK file
intFileListCount = 0;
using (var srInFile = new StreamReader(pekFile.OpenRead()))
{
while (!srInFile.EndOfStream)
{
var strLineIn = srInFile.ReadLine();
if ((strLineIn != null))
{
if (strLineIn.StartsWith(PEK_FILE_FILENAME_LINE))
{
intFileListCount += 1;
}
}
}
}
blnSuccess = true;
}
catch (Exception)
{
// Error opening or parsing the PEK file
blnSuccess = false;
}
if (intFileListCount > datasetFileInfo.ScanCount)
{
datasetFileInfo.ScanCount = intFileListCount;
}
// Only parse the first .Pek file found
break;
}
return(blnSuccess);
}
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
// Returns True if success, False if an error
// Override strDataFilePath here, if needed
// strDataFilePath = strDataFilePath;
// Obtain the full path to the file
var fiDatasetFile = new FileInfo(strDataFilePath);
datasetFileInfo.FileSystemCreationTime = fiDatasetFile.CreationTime;
datasetFileInfo.FileSystemModificationTime = fiDatasetFile.LastWriteTime;
// Using the file system modification time as the acquisition end time
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = 0;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(fiDatasetFile.Name);
datasetFileInfo.FileExtension = fiDatasetFile.Extension;
datasetFileInfo.FileSizeBytes = fiDatasetFile.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
mLCMS2DPlot.Options.UseObservedMinScan = false;
ProcessWiffFile(fiDatasetFile, datasetFileInfo);
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
UpdateDatasetFileStats(fiDatasetFile, datasetFileInfo.DatasetID);
// Copy over the updated filetime info and scan info from datasetFileInfo to mDatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
return(true);
}
private bool ParseBrukerAcquFile(string strFolderPath, clsDatasetFileInfo datasetFileInfo)
{
bool blnSuccess;
try
{
// Try to open the acqu file
blnSuccess = false;
using (var srInFile = new StreamReader(Path.Combine(strFolderPath, BRUKER_ACQU_FILE)))
{
while (!srInFile.EndOfStream)
{
var strLineIn = srInFile.ReadLine();
if ((strLineIn != null))
{
if (strLineIn.StartsWith(BRUKER_ACQU_FILE_ACQ_LINE_START))
{
// Date line found
// It is of the form: ##$AQ_DATE= <Sat Aug 20 07:56:55 2005>
strLineIn = strLineIn.Substring(BRUKER_ACQU_FILE_ACQ_LINE_START.Length).Trim();
strLineIn = strLineIn.TrimEnd(BRUKER_ACQU_FILE_ACQ_LINE_END);
blnSuccess = ParseBrukerDateFromArray(strLineIn, out var dtDate);
if (blnSuccess)
{
datasetFileInfo.AcqTimeEnd = dtDate;
}
break;
}
}
}
}
}
catch (Exception)
{
// Error opening the acqu file
blnSuccess = false;
}
return(blnSuccess);
}
private void ParseAnalysisCDFFile(string strFolderPath, clsDatasetFileInfo datasetFileInfo)
{
NetCDFReader.clsMSNetCdf objNETCDFReader = null;
try
{
// Note: as of May 2016 this only works if you compile as x86 or with "Prefer 32-bit" enabled when compiling as AnyCPU
objNETCDFReader = new NetCDFReader.clsMSNetCdf();
var blnSuccess = objNETCDFReader.OpenMSCdfFile(Path.Combine(strFolderPath, AGILENT_ANALYSIS_CDF_FILE));
if (blnSuccess)
{
var intScanCount = objNETCDFReader.GetScanCount();
if (intScanCount > 0)
{
// Lookup the scan time of the final scan
if (objNETCDFReader.GetScanInfo(intScanCount - 1, out var intScanNumber,
out _, out var dblScanTime, out _, out _))
{
// Add 1 to intScanNumber since the scan number is off by one in the CDF file
datasetFileInfo.ScanCount = intScanNumber + 1;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.Add(SecondsToTimeSpan(dblScanTime));
}
}
else
{
datasetFileInfo.ScanCount = 0;
}
}
}
catch (Exception ex)
{
OnWarningEvent("Error in ParseAnalysisCDFFile: " + ex.Message);
}
finally
{
objNETCDFReader?.CloseMSCdfFile();
}
}
private bool ParseBrukerLockFile(string strFolderPath, clsDatasetFileInfo datasetFileInfo)
{
bool blnSuccess;
try
{
// Try to open the Lock file
// The date line is the first (and only) line in the file
blnSuccess = false;
using (var srInFile = new StreamReader(Path.Combine(strFolderPath, BRUKER_LOCK_FILE)))
{
if (!srInFile.EndOfStream)
{
var strLineIn = srInFile.ReadLine();
if ((strLineIn != null))
{
// Date line found
// It is of the form: wd37119 2208 WD37119\9TOperator Sat Aug 20 06:10:31 2005
blnSuccess = ParseBrukerDateFromArray(strLineIn, out var dtDate);
if (blnSuccess)
{
datasetFileInfo.AcqTimeStart = dtDate;
}
}
}
}
}
catch (Exception)
{
// Error opening the Lock file
blnSuccess = false;
}
return(blnSuccess);
}
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
// Returns True if success, False if an error
DataReader objUIMFReader = null;
GlobalParams objGlobalParams = null;
var intMasterFrameNumList = new int[1];
// Obtain the full path to the file
var fiFileInfo = new FileInfo(strDataFilePath);
if (!fiFileInfo.Exists)
{
return(false);
}
// Future, optional: Determine the DatasetID
// Unfortunately, this is not present in metadata.txt
// intDatasetID = LookupDatasetID(strDatasetName)
var intDatasetID = 0;
datasetFileInfo.FileSystemCreationTime = fiFileInfo.CreationTime;
datasetFileInfo.FileSystemModificationTime = fiFileInfo.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = intDatasetID;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(fiFileInfo.Name);
datasetFileInfo.FileExtension = fiFileInfo.Extension;
datasetFileInfo.FileSizeBytes = fiFileInfo.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
var blnReadError = false;
var blnInaccurateStartTime = false;
try
{
// Use the UIMFLibrary to read the .UIMF file
objUIMFReader = new DataReader(fiFileInfo.FullName);
}
catch (Exception ex)
{
// File open failed
OnErrorEvent("Call to .OpenUIMF failed for " + fiFileInfo.Name + ": " + ex.Message, ex);
blnReadError = true;
}
if (!blnReadError)
{
try
{
// First obtain the global parameters
objGlobalParams = objUIMFReader.GetGlobalParams();
}
catch (Exception)
{
// Read error
blnReadError = true;
}
}
if (!blnReadError)
{
// Read the file info
var dctMasterFrameList = new Dictionary <int, DataReader.FrameType>();
try
{
// Construct a master list of frame numbers and frame types
dctMasterFrameList = objUIMFReader.GetMasterFrameList();
if (dctMasterFrameList.Count > 0)
{
// Copy the frame numbers into an array so that we can assure it's sorted
intMasterFrameNumList = new int[dctMasterFrameList.Keys.Count];
dctMasterFrameList.Keys.CopyTo(intMasterFrameNumList, 0);
Array.Sort(intMasterFrameNumList);
}
// Extract the acquisition time information
// The Global_Parameters table records the start time of the entire dataset in field DateStarted
// The Frame_Parameters table records the start time of reach frame in field StartTime
// The DateStarted column in the Global_Parameters table should be represented by one of these values
// A text-based date, like "5/2/2011 4:26:59 PM"; example: Sarc_MS2_26_2Apr11_Cheetah_11-02-18_inverse.uimf
// A text-based date (no time info), like "Thursday, January 13, 2011"; example: QC_Shew_11_01_pt5_c2_030311_earth_4ms_0001
// A tick-based date, like 129272890050787740 (number of ticks since January 1, 1601); example: BATs_TS_01_c4_Eagle_10-02-06_0000
// The StartTime column in the Frame_Parameters table should be represented by one of these values
// Integer between 0 and 1440 representing number of minutes since midnight (can loop from 1439.9 to 0); example: Sarc_MS2_26_2Apr11_Cheetah_11-02-18_inverse.uimf
// Integer between 0 and 60 representing number of minutes since past the current hour (can loop from 59.9 to 0); example: BATs_TS_01_c4_Eagle_10-02-06_0000.uimf
// A tick-based date, like 634305349108974800 (number of ticks since January 1, 0001); example: QC_Shew_11_01_pt5_c2_030311_earth_4ms_0001
// A negative number representing number of minutes from the start of the run in UTC time to the start of the current frame, in local time; example: Sarc_P08_G03_0747_7Dec11_Cheetah_11-09-05.uimf
// Examine values: Frame 1 has StartTime = -479.993 and Frame 1177 has StartTime = -417.509
// A positive integer representing number of minutes since the start of the run
// Theoretically, this will be the case for IMS_TOF_4 acquired after 12/14/2011
double dblRunTime = 0;
// First examine objGlobalParams.DateStarted
try
{
var blnValidStartTime = false;
var strReportedDateStarted = objGlobalParams.GetValue(GlobalParamKeyType.DateStarted);
if (!DateTime.TryParse(strReportedDateStarted, out var dtReportedDateStarted))
{
// Invalid date; log a message
OnWarningEvent(".UIMF file has an invalid DateStarted value in table Global_Parameters: " + strReportedDateStarted + "; " +
"will use the time the datafile was last modified");
blnInaccurateStartTime = true;
}
else
{
if (dtReportedDateStarted.Year < 450)
{
// Some .UIMF files have DateStarted values represented by huge integers, e.g. 127805472000000000 or 129145004045937500; example: BATs_TS_01_c4_Eagle_10-02-06_0000
// These numbers are the number of ticks since 1 January 1601 (where each tick is 100 ns)
// This value is returned by function GetSystemTimeAsFileTime (see http://en.wikipedia.org/wiki/System_time)
// When SQLite parses these numbers, it converts them to years around 0410
// To get the correct year, simply add 1600
dtReportedDateStarted = dtReportedDateStarted.AddYears(1600);
blnValidStartTime = true;
}
else if (dtReportedDateStarted.Year <2000 | dtReportedDateStarted.Year> DateTime.Now.Year + 1)
{
// Invalid date; log a message
OnWarningEvent(".UIMF file has an invalid DateStarted value in table Global_Parameters: " + strReportedDateStarted + "; " +
"will use the time the datafile was last modified");
blnInaccurateStartTime = true;
}
else
{
blnValidStartTime = true;
}
}
if (blnValidStartTime)
{
datasetFileInfo.AcqTimeStart = dtReportedDateStarted;
// Update the end time to match the start time; we'll update it below using the start/end times obtained from the frame parameters
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
}
catch (Exception ex2)
{
OnWarningEvent("Exception extracting the DateStarted date from table Global_Parameters in the .UIMF file: " + ex2.Message);
}
// NumFrames is the total number of frames in the file (for all frame types)
datasetFileInfo.ScanCount = objGlobalParams.NumFrames;
if (intMasterFrameNumList.Length > datasetFileInfo.ScanCount)
{
datasetFileInfo.ScanCount = intMasterFrameNumList.Length;
}
if (intMasterFrameNumList.Length > 0)
{
// Ideally, we would lookup the acquisition time of the first frame and the last frame, then subtract the two times to determine the run time
// However, given the odd values that can be present in the StartTime field, we need to construct a full list of start times and then parse it
// Get the start time of the first frame
// See above for the various numbers that could be stored in the StartTime column
var objFrameParams = objUIMFReader.GetFrameParams(intMasterFrameNumList[0]);
var dblStartTime = objFrameParams.GetValueDouble(FrameParamKeyType.StartTimeMinutes);
// Get the start time of the last frame
// If the reported start time is zero, then step back until a non-zero start time is reported
var frameIndex = intMasterFrameNumList.Length - 1;
double dblEndTime;
do
{
objFrameParams = objUIMFReader.GetFrameParams(intMasterFrameNumList[frameIndex]);
dblEndTime = objFrameParams.GetValueDouble(FrameParamKeyType.StartTimeMinutes);
if (Math.Abs(dblEndTime) < float.Epsilon)
{
frameIndex -= 1;
}
} while (Math.Abs(dblEndTime) < float.Epsilon && frameIndex >= 0);
// Check whether the StartTime and EndTime values are based on ticks
if (dblStartTime >= 1E+17 & dblEndTime > 1E+17)
{
// StartTime and Endtime were stored as the number of ticks (where each tick is 100 ns)
// Tick start date is either 1 January 1601 or 1 January 0001
var dtRunTime = DateTime.MinValue.AddTicks((long)(dblEndTime - dblStartTime));
dblRunTime = dtRunTime.Subtract(DateTime.MinValue).TotalMinutes;
// In some .UIMF files, the DateStarted column in Global_Parameters is simply the date, and not a specific time of day
// If that's the case, then update datasetFileInfo.AcqTimeStart to be based on dblRunTime
if (datasetFileInfo.AcqTimeStart.Date == datasetFileInfo.AcqTimeStart)
{
var dtReportedDateStarted = DateTime.MinValue.AddTicks((long)dblStartTime);
if (dtReportedDateStarted.Year < 500)
{
dtReportedDateStarted = dtReportedDateStarted.AddYears(1600);
}
if (dtReportedDateStarted.Year >= 2000 & dtReportedDateStarted.Year <= DateTime.Now.Year + 1)
{
// Date looks valid
if (blnInaccurateStartTime)
{
datasetFileInfo.AcqTimeStart = dtReportedDateStarted;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
else
{
// How does it compare to datasetFileInfo.AcqTimeStart?
if (dtReportedDateStarted.Subtract(datasetFileInfo.AcqTimeStart).TotalHours < 24)
{
// Update the date
datasetFileInfo.AcqTimeStart = dtReportedDateStarted;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
}
}
}
}
else
{
// Ideally, we'd just compute RunTime like this: dblRunTime = dblEndTime - dblStartTime
// But, given the idiosyncracies that can occur, we need to construct a full list of start times
var lstStartTimes = new List <double>();
double dblEndTimeAddon = 0;
for (var intIndex = 0; intIndex <= intMasterFrameNumList.Length - 1; intIndex++)
{
objFrameParams = objUIMFReader.GetFrameParams(intMasterFrameNumList[intIndex]);
lstStartTimes.Add(objFrameParams.GetValueDouble(FrameParamKeyType.StartTimeMinutes));
}
// Some datasets erroneously have zeroes stored in the .UIMF file for the StartTime of the last two frames; example: Sarc_MS2_26_2Apr11_Cheetah_11-02-18_inverse
// Check for this and remove them
var intFrameCountRemoved = 0;
while ((Math.Abs(lstStartTimes[lstStartTimes.Count - 1]) < float.Epsilon))
{
lstStartTimes.RemoveAt(lstStartTimes.Count - 1);
intFrameCountRemoved += 1;
if (lstStartTimes.Count == 0)
{
break;
}
}
if (intFrameCountRemoved > 0)
{
if (lstStartTimes.Count > 2)
{
// Compute the amount of time (in minutes) to addon to the total run time
// We're computing the time between two frames, and multiplying that by intFrameCountRemoved
dblEndTimeAddon += intFrameCountRemoved * (lstStartTimes[lstStartTimes.Count - 1] - lstStartTimes[lstStartTimes.Count - 2]);
}
}
// Now check for the StartTime changing to a smaller number from one frame to the next
// This could happen if the StartTime changed from 1439 to 0 as the system clock hits midnight
// Or if the StartTime changes from 59.9 to 0 as the system clock hits the top of a new hour
for (var intIndex = 1; intIndex <= lstStartTimes.Count - 1; intIndex++)
{
if (lstStartTimes[intIndex] < lstStartTimes[intIndex - 1])
{
if (lstStartTimes[intIndex - 1] > 1439)
{
dblEndTimeAddon += 1440;
}
else if (lstStartTimes[intIndex - 1] > 59.7)
{
dblEndTimeAddon += 60;
}
}
}
if (lstStartTimes.Count > 0)
{
// Compute the runtime
// Luckily, even if dblStartTime is -479.993 and dblEntTime is -417.509, this works out to a positive, accurate runtime
dblEndTime = lstStartTimes[lstStartTimes.Count - 1];
dblRunTime = dblEndTime + dblEndTimeAddon - dblStartTime;
}
}
}
else
{
dblRunTime = 0;
}
if (dblRunTime > 0)
{
if (dblRunTime > 24000)
{
OnWarningEvent("Invalid runtime computed using the StartTime value from the first and last frames: " + dblRunTime);
}
else
{
if (blnInaccurateStartTime)
{
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd.AddMinutes(-dblRunTime);
}
else
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(dblRunTime);
}
}
}
}
catch (Exception ex)
{
OnWarningEvent("Exception extracting acquisition time information: " + ex.Message);
}
if (mSaveTICAndBPI || mCreateDatasetInfoFile || mCreateScanStatsFile || mSaveLCMS2DPlots)
{
// Load data from each frame
// This is used to create the TIC and BPI plot, the 2D LC/MS plot, and/or to create the Dataset Info File
LoadFrameDetails(objUIMFReader, dctMasterFrameList, intMasterFrameNumList);
}
if (mComputeOverallQualityScores)
{
// Note that this call will also create the TICs and BPIs
ComputeQualityScores(objUIMFReader, datasetFileInfo, dctMasterFrameList, intMasterFrameNumList);
}
}
// Close the handle to the data file
objUIMFReader?.Dispose();
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
UpdateDatasetFileStats(fiFileInfo, intDatasetID);
// Copy over the updated filetime info from datasetFileInfo to mDatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.FileSystemCreationTime = datasetFileInfo.FileSystemCreationTime;
mDatasetStatsSummarizer.DatasetFileInfo.FileSystemModificationTime = datasetFileInfo.FileSystemModificationTime;
mDatasetStatsSummarizer.DatasetFileInfo.DatasetID = datasetFileInfo.DatasetID;
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
return(!blnReadError);
}
/// <summary>
/// This function is used to determine one or more overall quality scores
/// </summary>
/// <param name="objUIMFReader"></param>
/// <param name="datasetFileInfo"></param>
/// <param name="dctMasterFrameList"></param>
/// <param name="intMasterFrameNumList"></param>
private void ComputeQualityScores(
DataReader objUIMFReader,
clsDatasetFileInfo datasetFileInfo,
IReadOnlyDictionary <int, DataReader.FrameType> dctMasterFrameList,
IReadOnlyList <int> intMasterFrameNumList)
{
float sngOverallScore;
double dblOverallAvgIntensitySum = 0;
var intOverallAvgCount = 0;
if (mLCMS2DPlot.ScanCountCached > 0)
{
// Obtain the overall average intensity value using the data cached in mLCMS2DPlot
// This avoids having to reload all of the data using objUIMFReader
const int intMSLevelFilter = 1;
sngOverallScore = mLCMS2DPlot.ComputeAverageIntensityAllScans(intMSLevelFilter);
}
else
{
var objGlobalParams = objUIMFReader.GetGlobalParams();
var intGlobalMaxBins = objGlobalParams.Bins;
var dblMZList = new double[intGlobalMaxBins + 1];
var intIntensityList = new int[intGlobalMaxBins + 1];
// Call .GetStartAndEndScans to get the start and end Frames
GetStartAndEndScans(objGlobalParams.NumFrames, out var intFrameStart, out var intFrameEnd);
for (var intMasterFrameNumIndex = 0; intMasterFrameNumIndex <= intMasterFrameNumList.Count - 1; intMasterFrameNumIndex++)
{
var intFrameNumber = intMasterFrameNumList[intMasterFrameNumIndex];
if (!dctMasterFrameList.TryGetValue(intFrameNumber, out var eFrameType))
{
OnWarningEvent(string.Format(
"Frametype {0} not found in dictionary dctMasterFrameList; ignoring frame {1} in ComputeQualityScores",
eFrameType, intFrameNumber));
continue;
}
// Check whether the frame number is within the desired range
if (intFrameNumber < intFrameStart || intFrameNumber > intFrameEnd)
{
continue;
}
FrameParams objFrameParams;
try
{
objFrameParams = objUIMFReader.GetFrameParams(intFrameNumber);
}
catch (Exception)
{
Console.WriteLine("Exception obtaining frame parameters for frame " + intFrameNumber + "; will skip this frame");
objFrameParams = null;
}
if (objFrameParams == null)
{
continue;
}
// We have to clear the m/z and intensity arrays before calling GetSpectrum
Array.Clear(dblMZList, 0, dblMZList.Length);
Array.Clear(intIntensityList, 0, intIntensityList.Length);
// Process all of the IMS scans in this Frame to compute a summed spectrum representative of the frame
// Scans likely range from 0 to objFrameParams.Scans-1, but we'll use objFrameParams.Scans just to be safe
var intIonCount = objUIMFReader.GetSpectrum(intFrameNumber, intFrameNumber, eFrameType, 0, objFrameParams.Scans, out dblMZList, out intIntensityList);
if (intIonCount <= 0)
{
continue;
}
// The m/z and intensity arrays might contain entries with m/z values of 0;
// need to copy the data in place to get the data in the correct format.
if (intIonCount > dblMZList.Length)
{
intIonCount = dblMZList.Length;
}
var intTargetIndex = 0;
for (var intIonIndex = 0; intIonIndex <= intIonCount - 1; intIonIndex++)
{
if (dblMZList[intIonIndex] > 0)
{
if (intTargetIndex != intIonIndex)
{
dblMZList[intTargetIndex] = dblMZList[intIonIndex];
intIntensityList[intTargetIndex] = intIntensityList[intIonIndex];
}
intTargetIndex += 1;
}
}
intIonCount = intTargetIndex;
if (intIonCount > 0)
{
// ToDo: Analyze dblIonMZ and dblIonIntensity to compute a quality scores
// Keep track of the quality scores and then store one or more overall quality scores in datasetFileInfo.OverallQualityScore
// For now, this just computes the average intensity for each scan and then computes and overall average intensity value
double dblIntensitySum = 0;
for (var intIonIndex = 0; intIonIndex <= intIonCount - 1; intIonIndex++)
{
dblIntensitySum += intIntensityList[intIonIndex];
}
dblOverallAvgIntensitySum += dblIntensitySum / intIonCount;
intOverallAvgCount += 1;
}
}
if (intOverallAvgCount > 0)
{
sngOverallScore = (float)(dblOverallAvgIntensitySum / intOverallAvgCount);
}
else
{
sngOverallScore = 0;
}
}
datasetFileInfo.OverallQualityScore = sngOverallScore;
}
/// <summary>
/// Process the dataset
/// </summary>
/// <param name="strDataFilePath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error</returns>
/// <remarks></remarks>
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
var strDataFilePathLocal = string.Empty;
// Obtain the full path to the file
var fiRawFile = new FileInfo(strDataFilePath);
if (!fiRawFile.Exists)
{
OnErrorEvent(".Raw file not found: " + strDataFilePath);
return(false);
}
// Future, optional: Determine the DatasetID
// Unfortunately, this is not present in metadata.txt
// intDatasetID = LookupDatasetID(strDatasetName)
var intDatasetID = DatasetID;
// Record the file size and Dataset ID
datasetFileInfo.FileSystemCreationTime = fiRawFile.CreationTime;
datasetFileInfo.FileSystemModificationTime = fiRawFile.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = intDatasetID;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(fiRawFile.Name);
datasetFileInfo.FileExtension = fiRawFile.Extension;
datasetFileInfo.FileSizeBytes = fiRawFile.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
var blnDeleteLocalFile = false;
var blnReadError = false;
// Use Xraw to read the .Raw file
// If reading from a SAMBA-mounted network share, and if the current user has
// Read privileges but not Read&Execute privileges, then we will need to copy the file locally
var xcaliburAccessor = new XRawFileIO();
// Attach event handlers
xcaliburAccessor.ReportError += XcaliburAccessor_ReportError;
xcaliburAccessor.ReportWarning += XcaliburAccessor_ReportWarning;
// Open a handle to the data file
if (!xcaliburAccessor.OpenRawFile(fiRawFile.FullName))
{
// File open failed
OnErrorEvent("Call to .OpenRawFile failed for: " + fiRawFile.FullName);
blnReadError = true;
if (!string.Equals(clsMSFileInfoScanner.GetAppFolderPath().Substring(0, 2), fiRawFile.FullName.Substring(0, 2), StringComparison.InvariantCultureIgnoreCase))
{
if (mCopyFileLocalOnReadError)
{
// Copy the file locally and try again
try
{
strDataFilePathLocal = Path.Combine(clsMSFileInfoScanner.GetAppFolderPath(), Path.GetFileName(strDataFilePath));
if (!string.Equals(strDataFilePathLocal, strDataFilePath, StringComparison.InvariantCultureIgnoreCase))
{
OnDebugEvent("Copying file " + Path.GetFileName(strDataFilePath) + " to the working folder");
File.Copy(strDataFilePath, strDataFilePathLocal, true);
strDataFilePath = string.Copy(strDataFilePathLocal);
blnDeleteLocalFile = true;
// Update fiRawFile then try to re-open
fiRawFile = new FileInfo(strDataFilePath);
if (!xcaliburAccessor.OpenRawFile(fiRawFile.FullName))
{
// File open failed
OnErrorEvent("Call to .OpenRawFile failed for: " + fiRawFile.FullName);
blnReadError = true;
}
else
{
blnReadError = false;
}
}
}
catch (Exception)
{
blnReadError = true;
}
}
}
}
if (!blnReadError)
{
// Read the file info
try
{
datasetFileInfo.AcqTimeStart = xcaliburAccessor.FileInfo.CreationDate;
}
catch (Exception)
{
// Read error
blnReadError = true;
}
if (!blnReadError)
{
try
{
// Look up the end scan time then compute .AcqTimeEnd
var intScanEnd = xcaliburAccessor.FileInfo.ScanEnd;
xcaliburAccessor.GetScanInfo(intScanEnd, out clsScanInfo scanInfo);
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(scanInfo.RetentionTime);
datasetFileInfo.ScanCount = xcaliburAccessor.GetNumScans();
}
catch (Exception)
{
// Error; use default values
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
datasetFileInfo.ScanCount = 0;
}
if (mSaveTICAndBPI || mCreateDatasetInfoFile || mCreateScanStatsFile || mSaveLCMS2DPlots || mCheckCentroidingStatus)
{
// Load data from each scan
// This is used to create the TIC and BPI plot, the 2D LC/MS plot, and/or to create the Dataset Info File
LoadScanDetails(xcaliburAccessor);
}
if (mComputeOverallQualityScores)
{
// Note that this call will also create the TICs and BPIs
ComputeQualityScores(xcaliburAccessor, datasetFileInfo);
}
}
}
mDatasetStatsSummarizer.SampleInfo.SampleName = xcaliburAccessor.FileInfo.SampleName;
mDatasetStatsSummarizer.SampleInfo.Comment1 = xcaliburAccessor.FileInfo.Comment1;
mDatasetStatsSummarizer.SampleInfo.Comment2 = xcaliburAccessor.FileInfo.Comment2;
if (!string.IsNullOrEmpty(xcaliburAccessor.FileInfo.SampleComment))
{
if (string.IsNullOrEmpty(mDatasetStatsSummarizer.SampleInfo.Comment1))
{
mDatasetStatsSummarizer.SampleInfo.Comment1 = xcaliburAccessor.FileInfo.SampleComment;
}
else
{
if (string.IsNullOrEmpty(mDatasetStatsSummarizer.SampleInfo.Comment2))
{
mDatasetStatsSummarizer.SampleInfo.Comment2 = xcaliburAccessor.FileInfo.SampleComment;
}
else
{
// Append the sample comment to comment 2
mDatasetStatsSummarizer.SampleInfo.Comment2 += "; " + xcaliburAccessor.FileInfo.SampleComment;
}
}
}
// Close the handle to the data file
xcaliburAccessor.CloseRawFile();
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
UpdateDatasetFileStats(fiRawFile, intDatasetID);
// Copy over the updated filetime info from datasetFileInfo to mDatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.FileSystemCreationTime = datasetFileInfo.FileSystemCreationTime;
mDatasetStatsSummarizer.DatasetFileInfo.FileSystemModificationTime = datasetFileInfo.FileSystemModificationTime;
mDatasetStatsSummarizer.DatasetFileInfo.DatasetID = datasetFileInfo.DatasetID;
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
// Delete the local copy of the data file
if (blnDeleteLocalFile)
{
try
{
File.Delete(strDataFilePathLocal);
}
catch (Exception)
{
// Deletion failed
OnErrorEvent("Deletion failed for: " + Path.GetFileName(strDataFilePathLocal));
}
}
return(!blnReadError);
}
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
// Returns True if success, False if an error
var blnSuccess = false;
var blnAcqTimeDetermined = false;
try {
var ioFolderInfo = new DirectoryInfo(strDataFilePath);
var strMSDataFilePath = Path.Combine(ioFolderInfo.FullName, AGILENT_MS_DATA_FILE);
datasetFileInfo.FileSystemCreationTime = ioFolderInfo.CreationTime;
datasetFileInfo.FileSystemModificationTime = ioFolderInfo.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(ioFolderInfo.Name);
datasetFileInfo.FileExtension = ioFolderInfo.Extension;
datasetFileInfo.FileSizeBytes = 0;
// Look for the MS file
// Use its modification time to get an initial estimate for the acquisition end time
// Assign the .MS file's size to .FileSizeBytes
var fiMSDatafile = new FileInfo(strMSDataFilePath);
if (fiMSDatafile.Exists)
{
datasetFileInfo.FileSizeBytes = fiMSDatafile.Length;
datasetFileInfo.AcqTimeStart = fiMSDatafile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = fiMSDatafile.LastWriteTime;
// Read the file info from the file system
UpdateDatasetFileStats(fiMSDatafile, datasetFileInfo.DatasetID);
blnSuccess = true;
}
datasetFileInfo.ScanCount = 0;
if (blnSuccess)
{
// Read the detailed data from the MS file
blnSuccess = ProcessChemstationMSDataFile(strMSDataFilePath, datasetFileInfo);
if (blnSuccess)
{
blnAcqTimeDetermined = true;
}
}
if (!blnSuccess)
{
// MS file not found (or problems parsing); use acqmeth.txt and/or GC.ini
// The timestamp of the acqmeth.txt file or GC.ini file is more accurate than the GC.ini file, so we'll use that
var fiMethodFile = new FileInfo(Path.Combine(ioFolderInfo.FullName, AGILENT_ACQ_METHOD_FILE));
if (!fiMethodFile.Exists)
{
fiMethodFile = new FileInfo(Path.Combine(ioFolderInfo.FullName, AGILENT_GC_INI_FILE));
}
if (fiMethodFile.Exists)
{
// Update the AcqTimes only if the LastWriteTime of the acqmeth.txt or GC.ini file is within the next 60 minutes of .AcqTimeEnd
if (!blnSuccess || fiMethodFile.LastWriteTime.Subtract(datasetFileInfo.AcqTimeEnd).TotalMinutes < 60)
{
datasetFileInfo.AcqTimeStart = fiMethodFile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = fiMethodFile.LastWriteTime;
blnSuccess = true;
}
if (datasetFileInfo.FileSizeBytes == 0)
{
// File size was not determined from the MS file
// Instead, sum up the sizes of all of the files in this folder
foreach (var item in ioFolderInfo.GetFiles())
{
datasetFileInfo.FileSizeBytes += item.Length;
}
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
}
}
}
if (!blnAcqTimeDetermined)
{
try {
// Parse the acqmeth.txt file to determine the actual values for .AcqTimeStart and .AcqTimeEnd
blnSuccess = ParseAcqMethodFile(strDataFilePath, datasetFileInfo);
if (!blnSuccess)
{
// Try to extract Runtime from the GC.ini file
blnSuccess = ParseGCIniFile(strDataFilePath, datasetFileInfo);
}
} catch (Exception) {
// Error parsing the acqmeth.txt file or GC.in file; do not abort
}
// We set blnSuccess to true, even if either of the above functions fail
blnSuccess = true;
}
if (blnSuccess)
{
// Copy over the updated filetime info and scan info from datasetFileInfo to mDatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
}
} catch (Exception ex) {
OnErrorEvent("Exception parsing GC .D folder: " + ex.Message, ex);
blnSuccess = false;
}
return(blnSuccess);
}
private bool ParseAcqMethodFile(string strFolderPath, clsDatasetFileInfo datasetFileInfo)
{
double dblTotalRuntime = 0;
var blnRunTimeFound = false;
bool blnSuccess;
try {
// Open the acqmeth.txt file
var strFilePath = Path.Combine(strFolderPath, AGILENT_ACQ_METHOD_FILE);
if (!File.Exists(strFilePath))
{
return(false);
}
// Populate a dictionary var with the text strings for finding lines with runtime information
// Note that "Post Run" occurs twice in the file, so we use clsLineMatchSearchInfo.Matched to track whether or not the text has been matched
var dctRunTimeText = new Dictionary <string, clsLineMatchSearchInfo>
{
{ ACQ_METHOD_FILE_EQUILIBRATION_TIME_LINE, new clsLineMatchSearchInfo(true) },
{ ACQ_METHOD_FILE_RUN_TIME_LINE, new clsLineMatchSearchInfo(true) }
};
// We could also add in the "Post Run" time for determining total acquisition time, but we don't do this, to stay consistent with run times reported by the MS file
// dctRunTimeText.Add(ACQ_METHOD_FILE_POST_RUN_LINE, New clsLineMatchSearchInfo(False))
using (var srInFile = new StreamReader(strFilePath)) {
while (!srInFile.EndOfStream)
{
var strLineIn = srInFile.ReadLine();
if (string.IsNullOrWhiteSpace(strLineIn))
{
continue;
}
foreach (var strKey in dctRunTimeText.Keys)
{
if (dctRunTimeText[strKey].Matched)
{
continue;
}
bool blnMatchSuccess;
if (dctRunTimeText[strKey].MatchLineStart)
{
blnMatchSuccess = strLineIn.StartsWith(strKey);
}
else
{
blnMatchSuccess = strLineIn.Contains(strKey);
}
if (!blnMatchSuccess)
{
continue;
}
if (!ExtractRunTime(strLineIn, out var dblRunTime))
{
continue;
}
dctRunTimeText[strKey].Matched = true;
dblTotalRuntime += dblRunTime;
blnRunTimeFound = true;
break;
}
}
}
blnSuccess = blnRunTimeFound;
} catch (Exception ex) {
// Exception reading file
OnErrorEvent("Exception reading " + AGILENT_ACQ_METHOD_FILE + ": " + ex.Message, ex);
blnSuccess = false;
}
if (blnSuccess)
{
// Update the acquisition start time
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd.AddMinutes(-dblTotalRuntime);
}
return(blnSuccess);
}
private bool ParseBrukerZippedSFolders(DirectoryInfo diZippedSFilesFolderInfo, clsDatasetFileInfo datasetFileInfo)
{
// Looks through the s*.zip files to determine the total file size (uncompressed) of all files in all the matching .Zip files
// Updates datasetFileInfo.FileSizeBytes with this info, while also updating datasetFileInfo.ScanCount with the total number of files found
// Returns True if success and also if no matching Zip files were found; returns False if error
bool blnSuccess;
datasetFileInfo.FileSizeBytes = 0;
datasetFileInfo.ScanCount = 0;
try
{
foreach (var zippedSFile in diZippedSFilesFolderInfo.GetFiles("s*.zip"))
{
// Get the info on each zip file
using (var objZipFile = new Ionic.Zip.ZipFile(zippedSFile.FullName))
{
foreach (var objZipEntry in objZipFile.Entries)
{
datasetFileInfo.FileSizeBytes += objZipEntry.UncompressedSize;
datasetFileInfo.ScanCount += 1;
}
}
}
blnSuccess = true;
}
catch (Exception)
{
blnSuccess = false;
}
return(blnSuccess);
}
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
// Returns True if success, False if an error
var blnSuccess = false;
try
{
var diRootFolder = new DirectoryInfo(strDataFilePath);
var diAcqDataFolder = new DirectoryInfo(Path.Combine(diRootFolder.FullName, AGILENT_ACQDATA_FOLDER_NAME));
datasetFileInfo.FileSystemCreationTime = diAcqDataFolder.CreationTime;
datasetFileInfo.FileSystemModificationTime = diAcqDataFolder.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(diRootFolder.Name);
datasetFileInfo.FileExtension = diRootFolder.Extension;
datasetFileInfo.FileSizeBytes = 0;
datasetFileInfo.ScanCount = 0;
if (diAcqDataFolder.Exists)
{
// Sum up the sizes of all of the files in the AcqData folder
foreach (var fiFile in diAcqDataFolder.GetFiles("*", SearchOption.AllDirectories))
{
datasetFileInfo.FileSizeBytes += fiFile.Length;
}
// Look for the MSScan.bin file
// Use its modification time to get an initial estimate for the acquisition end time
var fiMSScanfile = new FileInfo(Path.Combine(diAcqDataFolder.FullName, AGILENT_MS_SCAN_FILE));
if (fiMSScanfile.Exists)
{
datasetFileInfo.AcqTimeStart = fiMSScanfile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = fiMSScanfile.LastWriteTime;
// Read the file info from the file system
// Several of these stats will be further updated later
UpdateDatasetFileStats(fiMSScanfile, datasetFileInfo.DatasetID);
}
else
{
// Read the file info from the file system
// Several of these stats will be further updated later
UpdateDatasetFileStats(diAcqDataFolder, datasetFileInfo.DatasetID);
}
blnSuccess = true;
}
if (blnSuccess)
{
// The AcqData folder exists
// Parse the Contents.xml file to determine the acquisition start time
var blnAcqStartTimeDetermined = ProcessContentsXMLFile(diAcqDataFolder.FullName, datasetFileInfo);
// Parse the MSTS.xml file to determine the acquisition length and number of scans
var blnValidMSTS = ProcessTimeSegmentFile(diAcqDataFolder.FullName, datasetFileInfo, out var dblAcquisitionLengthMinutes);
if (!blnAcqStartTimeDetermined && blnValidMSTS)
{
// Compute the start time from .AcqTimeEnd minus dblAcquisitionLengthMinutes
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd.AddMinutes(-dblAcquisitionLengthMinutes);
}
// Note: could parse the AcqMethod.xml file to determine if MS2 spectra are present
//<AcqMethod>
// <QTOF>
// <TimeSegment>
// <Acquisition>
// <AcqMode>TargetedMS2</AcqMode>
// Read the raw data to create the TIC and BPI
ReadBinaryData(diRootFolder.FullName, datasetFileInfo);
}
if (blnSuccess)
{
// Copy over the updated filetime info and scan info from datasetFileInfo to mDatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
}
}
catch (Exception ex)
{
OnErrorEvent("Exception parsing Agilent TOF .D folder: " + ex.Message, ex);
blnSuccess = false;
}
return(blnSuccess);
}
private void LoadData(FileInfo fiIsosFile, clsDatasetFileInfo datasetFileInfo)
{
// Cache the data in the _isos.csv and _scans.csv files
if (mSaveTICAndBPI)
{
// Initialize the TIC and BPI arrays
InitializeTICAndBPI();
}
if (mSaveLCMS2DPlots)
{
InitializeLCMS2DPlot();
}
var lstIsosData = LoadIsosFile(fiIsosFile.FullName, MaxFit);
if (lstIsosData.Count == 0)
{
OnErrorEvent("No data found in the _isos.csv file: " + fiIsosFile.FullName);
return;
}
var strScansFilePath = GetDatasetNameViaPath(fiIsosFile.Name) + DECONTOOLS_SCANS_FILE_SUFFIX;
if (fiIsosFile.Directory != null)
{
strScansFilePath = Path.Combine(fiIsosFile.Directory.FullName, strScansFilePath);
}
var lstScanData = LoadScansFile(strScansFilePath);
var scansFileIsMissing = false;
if (lstScanData.Count > 0)
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(lstScanData.Last().ElutionTime);
datasetFileInfo.ScanCount = lstScanData.Last().Scan;
}
else
{
scansFileIsMissing = true;
datasetFileInfo.ScanCount = (from item in lstIsosData select item.Scan).Max();
for (var intScanIndex = 1; intScanIndex <= datasetFileInfo.ScanCount; intScanIndex++)
{
var udtScanData = new udtScansDataType
{
Scan = intScanIndex,
ElutionTime = intScanIndex,
MSLevel = 1
};
lstScanData.Add(udtScanData);
}
}
// Step through the isos data and call mLCMS2DPlot.AddScan() for each scan
var lstIons = new List <clsLCMSDataPlotter.udtMSIonType>();
var intCurrentScan = 0;
// Note: we only need to update mLCMS2DPlot
// The options for mLCMS2DPlotOverview will be cloned from mLCMS2DPlot.Options
mLCMS2DPlot.Options.PlottingDeisotopedData = true;
var dblMaxMonoMass = mLCMS2DPlot.Options.MaxMonoMassForDeisotopedPlot;
for (var intIndex = 0; intIndex <= lstIsosData.Count - 1; intIndex++)
{
if (lstIsosData[intIndex].Scan > intCurrentScan || intIndex == lstIsosData.Count - 1)
{
// Store the cached values
if (lstIons.Count > 0)
{
var udtCurrentScan = (from item in lstScanData where item.Scan == intCurrentScan select item).ToList().FirstOrDefault();
lstIons.Sort(new clsLCMSDataPlotter.udtMSIonTypeComparer());
mLCMS2DPlot.AddScan(intCurrentScan, udtCurrentScan.MSLevel, udtCurrentScan.ElutionTime, lstIons);
if (scansFileIsMissing && mSaveTICAndBPI)
{
// Determine the TIC and BPI values using the data from the .isos file
double tic = 0;
double bpi = 0;
for (var dataIndex = 0; dataIndex <= lstIons.Count - 1; dataIndex++)
{
tic += lstIons[dataIndex].Intensity;
if (lstIons[dataIndex].Intensity > bpi)
{
bpi = lstIons[dataIndex].Intensity;
}
}
mTICandBPIPlot.AddData(intCurrentScan, udtCurrentScan.MSLevel, udtCurrentScan.ElutionTime, bpi, tic);
}
}
intCurrentScan = lstIsosData[intIndex].Scan;
lstIons.Clear();
}
if (lstIsosData[intIndex].MonoMass <= dblMaxMonoMass)
{
var udtIon = new clsLCMSDataPlotter.udtMSIonType
{
// Note that we store .MonoMass in a field called .mz; we'll still be plotting monoisotopic mass
MZ = lstIsosData[intIndex].MonoMass,
Intensity = lstIsosData[intIndex].Abundance,
Charge = lstIsosData[intIndex].Charge
};
lstIons.Add(udtIon);
}
}
}
/// <summary>
/// Examines the subdirectories in the specified zip file
/// Determines the oldest and newest modified analysis.baf files (or apexAcquisition.method file if analysis.baf files are not found)
/// Presumes this is the AcqStartTime and AcqEndTime
/// </summary>
/// <param name="fiZipFile"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if at least one valid file is found; otherwise false</returns>
/// <remarks></remarks>
private void DetermineAcqStartEndTime(FileInfo fiZipFile, clsDatasetFileInfo datasetFileInfo)
{
var blnSuccess = false;
try
{
// Bump up the file size
datasetFileInfo.FileSizeBytes += fiZipFile.Length;
var lstFileNamesToFind = new List <string> {
"analysis.baf",
"apexAcquisition.method",
"submethods.xml"
};
var oZipFile = new Ionic.Zip.ZipFile(fiZipFile.FullName);
foreach (var strFileNameToFind in lstFileNamesToFind)
{
using (var oZipEntry = oZipFile.GetEnumerator())
{
while (oZipEntry.MoveNext())
{
if (oZipEntry.Current == null)
{
continue;
}
if (oZipEntry.Current.IsDirectory)
{
continue;
}
// Split the filename on the forward slash character
var strNameParts = oZipEntry.Current.FileName.Split('/');
if (strNameParts.Length <= 0)
{
continue;
}
if (!string.Equals(strNameParts[strNameParts.Length - 1], strFileNameToFind, StringComparison.CurrentCultureIgnoreCase))
{
continue;
}
if (oZipEntry.Current.LastModified < datasetFileInfo.AcqTimeStart)
{
datasetFileInfo.AcqTimeStart = oZipEntry.Current.LastModified;
}
if (oZipEntry.Current.LastModified > datasetFileInfo.AcqTimeEnd)
{
datasetFileInfo.AcqTimeEnd = oZipEntry.Current.LastModified;
}
// Bump up the scan count
datasetFileInfo.ScanCount += 1;
// Add a Scan Stats entry
var objScanStatsEntry = new clsScanStatsEntry
{
ScanNumber = datasetFileInfo.ScanCount,
ScanType = 1,
ScanTypeName = "MALDI-HMS",
ScanFilterText = "",
ElutionTime = "0",
TotalIonIntensity = "0",
BasePeakIntensity = "0",
BasePeakMZ = "0",
BasePeakSignalToNoiseRatio = "0",
IonCount = 0,
IonCountRaw = 0
};
// Base peak signal to noise ratio
mDatasetStatsSummarizer.AddDatasetScan(objScanStatsEntry);
blnSuccess = true;
}
}
if (blnSuccess)
{
break;
}
}
}
catch (Exception ex)
{
OnErrorEvent("Error in DetermineAcqStartEndTime: " + ex.Message);
}
}
private bool ParseTICFolder(DirectoryInfo ticFolderInfo, clsDatasetFileInfo datasetFileInfo, out DateTime dtTICModificationDate)
{
// Look for and open the .Tic file in ioFolderInfo and look for the line listing the number of files
// As a second validation, count the number of lines between TIC_FILE_TIC_FILE_LIST_START and TIC_FILE_TIC_FILE_LIST_END
var intFileListCount = 0;
var blnParsingTICFileList = false;
var blnSuccess = false;
dtTICModificationDate = DateTime.MinValue;
foreach (var ticFile in ticFolderInfo.GetFiles("*.tic"))
{
try
{
// Try to open the TIC file
intFileListCount = 0;
using (var srInFile = new StreamReader(ticFile.OpenRead()))
{
while (!srInFile.EndOfStream)
{
var strLineIn = srInFile.ReadLine();
if ((string.IsNullOrEmpty(strLineIn)))
{
continue;
}
if (blnParsingTICFileList)
{
if (strLineIn.StartsWith(TIC_FILE_TIC_FILE_LIST_END))
{
blnParsingTICFileList = false;
break;
}
if (strLineIn == TIC_FILE_COMMENT_SECTION_END)
{
// Found the end of the text section; exit the loop
break;
}
intFileListCount += 1;
}
else
{
if (strLineIn.StartsWith(TIC_FILE_NUMBER_OF_FILES_LINE_START))
{
// Number of files line found
// Parse out the file count
datasetFileInfo.ScanCount = int.Parse(strLineIn.Substring(TIC_FILE_NUMBER_OF_FILES_LINE_START.Length).Trim());
}
else if (strLineIn.StartsWith(TIC_FILE_TIC_FILE_LIST_START))
{
blnParsingTICFileList = true;
}
else if (strLineIn == TIC_FILE_COMMENT_SECTION_END)
{
// Found the end of the text section; exit the loop
break;
}
}
}
}
blnSuccess = true;
dtTICModificationDate = ticFile.LastWriteTime;
}
catch (Exception)
{
// Error opening or parsing the TIC file
blnSuccess = false;
}
if (intFileListCount > datasetFileInfo.ScanCount)
{
datasetFileInfo.ScanCount = intFileListCount;
}
// Only parse the first .Tic file found
break;
}
return(blnSuccess);
}
/// <summary>
///
/// </summary>
/// <param name="strDataFilePath">Bruker 1 folder path or Bruker s001.zip file</param>
/// <param name="datasetFileInfo"></param>
/// <returns></returns>
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
// Process a Bruker 1 folder or Bruker s001.zip file, specified by strDataFilePath
// If a Bruker 1 folder, it must contain file acqu and typically contains file LOCK
DirectoryInfo diZippedSFilesFolderInfo = null;
var intScanCountSaved = 0;
var dtTICModificationDate = DateTime.MinValue;
var blnParsingBrukerOneFolder = false;
bool blnSuccess;
try
{
// Determine whether strDataFilePath points to a file or a folder
// See if strFileOrFolderPath points to a valid file
var brukerDatasetfile = new FileInfo(strDataFilePath);
if (brukerDatasetfile.Exists)
{
// Parsing a zipped S folder
blnParsingBrukerOneFolder = false;
// The dataset name is equivalent to the name of the folder containing strDataFilePath
diZippedSFilesFolderInfo = brukerDatasetfile.Directory;
blnSuccess = true;
// Cannot determine accurate acqusition start or end times
// We have to assign a date, so we'll assign the date for the zipped s-folder
datasetFileInfo.AcqTimeStart = brukerDatasetfile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = brukerDatasetfile.LastWriteTime;
}
else
{
// Assuming it's a "1" folder
blnParsingBrukerOneFolder = true;
diZippedSFilesFolderInfo = new DirectoryInfo(strDataFilePath);
if (diZippedSFilesFolderInfo.Exists)
{
// Determine the dataset name by looking up the name of the parent folder of strDataFilePath
diZippedSFilesFolderInfo = diZippedSFilesFolderInfo.Parent;
blnSuccess = true;
}
else
{
blnSuccess = false;
}
}
if (blnSuccess && diZippedSFilesFolderInfo != null)
{
datasetFileInfo.FileSystemCreationTime = diZippedSFilesFolderInfo.CreationTime;
datasetFileInfo.FileSystemModificationTime = diZippedSFilesFolderInfo.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetName = diZippedSFilesFolderInfo.Name;
datasetFileInfo.FileExtension = string.Empty;
datasetFileInfo.FileSizeBytes = 0;
datasetFileInfo.ScanCount = 0;
}
}
catch (Exception)
{
blnSuccess = false;
}
if (blnSuccess && blnParsingBrukerOneFolder)
{
// Parse the Acqu File to populate .AcqTimeEnd
blnSuccess = ParseBrukerAcquFile(strDataFilePath, datasetFileInfo);
if (blnSuccess)
{
// Parse the Lock file to populate.AcqTimeStart
blnSuccess = ParseBrukerLockFile(strDataFilePath, datasetFileInfo);
if (!blnSuccess)
{
// Use the end time as the start time
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd;
blnSuccess = true;
}
}
}
if (blnSuccess)
{
// Look for the zipped S folders in ioZippedSFilesFolderInfo
try
{
blnSuccess = ParseBrukerZippedSFolders(diZippedSFilesFolderInfo, datasetFileInfo);
intScanCountSaved = datasetFileInfo.ScanCount;
}
catch (Exception)
{
// Error parsing zipped S Folders; do not abort
}
try
{
blnSuccess = false;
// Look for the TIC* folder to obtain the scan count from a .Tic file
// If the Scan Count in the TIC is larger than the scan count from ParseBrukerZippedSFolders,
// then we'll use that instead
foreach (var subFolder in diZippedSFilesFolderInfo.GetDirectories("TIC*"))
{
blnSuccess = ParseTICFolder(subFolder, datasetFileInfo, out dtTICModificationDate);
if (blnSuccess)
{
// Successfully parsed a TIC folder; do not parse any others
break;
}
}
if (!blnSuccess)
{
// TIC folder not found; see if a .TIC file is present in ioZippedSFilesFolderInfo
blnSuccess = ParseTICFolder(diZippedSFilesFolderInfo, datasetFileInfo, out dtTICModificationDate);
}
if (blnSuccess & !blnParsingBrukerOneFolder && dtTICModificationDate >= MINIMUM_ACCEPTABLE_ACQ_START_TIME)
{
// If dtTICModificationDate is earlier than .AcqTimeStart then update to dtTICMOdificationDate
if (dtTICModificationDate < datasetFileInfo.AcqTimeStart)
{
datasetFileInfo.AcqTimeStart = dtTICModificationDate;
datasetFileInfo.AcqTimeEnd = dtTICModificationDate;
}
}
if (!blnSuccess)
{
// .Tic file not found in ioZippedSFilesFolderInfo
// Look for an ICR* folder to obtain the scan count from a .Pek file
foreach (var subFolder in diZippedSFilesFolderInfo.GetDirectories("ICR*"))
{
blnSuccess = ParseICRFolder(subFolder, datasetFileInfo);
if (blnSuccess)
{
// Successfully parsed an ICR folder; do not parse any others
break;
}
}
}
if (blnSuccess)
{
if (intScanCountSaved > datasetFileInfo.ScanCount)
{
datasetFileInfo.ScanCount = intScanCountSaved;
}
}
else
{
// Set success to true anyway since we do have enough information to save the MS file info
blnSuccess = true;
}
}
catch (Exception)
{
// Error parsing the TIC* or ICR* folders; do not abort
}
// Validate datasetFileInfo.AcqTimeStart vs. datasetFileInfo.AcqTimeEnd
if (datasetFileInfo.AcqTimeEnd >= MINIMUM_ACCEPTABLE_ACQ_START_TIME)
{
if (datasetFileInfo.AcqTimeStart > datasetFileInfo.AcqTimeEnd)
{
// Start time cannot be greater than the end time
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd;
}
else if (datasetFileInfo.AcqTimeStart < MINIMUM_ACCEPTABLE_ACQ_START_TIME)
{
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd;
}
}
}
return(blnSuccess);
}
private void ReadBinaryData(string strDataFolderPath, clsDatasetFileInfo datasetFileInfo)
{
try
{
// Open the data folder using the ProteoWizardWrapper
var objPWiz = new pwiz.ProteowizardWrapper.MSDataFileReader(strDataFolderPath);
try
{
var dtRunStartTime = Convert.ToDateTime(objPWiz.RunStartTime);
// Update AcqTimeEnd if possible
if (dtRunStartTime < datasetFileInfo.AcqTimeEnd)
{
if (datasetFileInfo.AcqTimeEnd.Subtract(dtRunStartTime).TotalDays < 1)
{
datasetFileInfo.AcqTimeStart = dtRunStartTime;
}
}
}
catch (Exception)
{
// Leave the times unchanged
}
// Instantiate the Proteowizard Data Parser class
var pWizParser = new clsProteowizardDataParser(objPWiz, mDatasetStatsSummarizer, mTICandBPIPlot,
mLCMS2DPlot, mSaveLCMS2DPlots, mSaveTICAndBPI,
mCheckCentroidingStatus)
{
HighResMS1 = true,
HighResMS2 = true
};
RegisterEvents(pWizParser);
var blnTICStored = false;
double dblRuntimeMinutes = 0;
if (objPWiz.ChromatogramCount > 0)
{
// Process the chromatograms
pWizParser.StoreChromatogramInfo(datasetFileInfo, out blnTICStored, out _, out dblRuntimeMinutes);
pWizParser.PossiblyUpdateAcqTimeStart(datasetFileInfo, dblRuntimeMinutes);
}
if (objPWiz.SpectrumCount > 0)
{
// Process the spectral data
pWizParser.StoreMSSpectraInfo(datasetFileInfo, blnTICStored, ref dblRuntimeMinutes);
pWizParser.PossiblyUpdateAcqTimeStart(datasetFileInfo, dblRuntimeMinutes);
}
objPWiz.Dispose();
clsProgRunner.GarbageCollectNow();
}
catch (Exception ex)
{
OnErrorEvent("Exception reading the Binary Data in the Agilent TOF .D folder using Proteowizard: " + ex.Message, ex);
}
}
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
// Returns True if success, False if an error
try {
var ioFolderInfo = new DirectoryInfo(strDataFilePath);
datasetFileInfo.FileSystemCreationTime = ioFolderInfo.CreationTime;
datasetFileInfo.FileSystemModificationTime = ioFolderInfo.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(ioFolderInfo.Name);
datasetFileInfo.FileExtension = ioFolderInfo.Extension;
// Sum up the sizes of all of the files in this folder
datasetFileInfo.FileSizeBytes = 0;
var intFileCount = 0;
foreach (var item in ioFolderInfo.GetFiles())
{
datasetFileInfo.FileSizeBytes += item.Length;
if (intFileCount == 0)
{
// Assign the first file's modification time to .AcqTimeStart and .AcqTimeEnd
// Necessary in case _header.txt is missing
datasetFileInfo.AcqTimeStart = item.LastWriteTime;
datasetFileInfo.AcqTimeEnd = item.LastWriteTime;
}
if (item.Name.ToLower() == "_header.txt")
{
// Assign the file's modification time to .AcqTimeStart and .AcqTimeEnd
// These will get updated below to more precise values
datasetFileInfo.AcqTimeStart = item.LastWriteTime;
datasetFileInfo.AcqTimeEnd = item.LastWriteTime;
}
intFileCount += 1;
}
datasetFileInfo.ScanCount = 0;
var objNativeFileIO = new clsMassLynxNativeIO();
if (objNativeFileIO.GetFileInfo(ioFolderInfo.FullName, out var udtHeaderInfo))
{
var dtNewStartDate = DateTime.Parse(udtHeaderInfo.AcquDate + " " + udtHeaderInfo.AcquTime);
var intFunctionCount = objNativeFileIO.GetFunctionCount(ioFolderInfo.FullName);
if (intFunctionCount > 0)
{
// Sum up the scan count of all of the functions
// Additionally, find the largest EndRT value in all of the functions
float sngEndRT = 0;
for (var intFunctionNumber = 1; intFunctionNumber <= intFunctionCount; intFunctionNumber++)
{
if (objNativeFileIO.GetFunctionInfo(ioFolderInfo.FullName, 1, out clsMassLynxNativeIO.udtMSFunctionInfoType udtFunctionInfo))
{
datasetFileInfo.ScanCount += udtFunctionInfo.ScanCount;
if (udtFunctionInfo.EndRT > sngEndRT)
{
sngEndRT = udtFunctionInfo.EndRT;
}
}
}
if (dtNewStartDate >= MINIMUM_ACCEPTABLE_ACQ_START_TIME)
{
datasetFileInfo.AcqTimeStart = dtNewStartDate;
if (sngEndRT > 0)
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.Add(MinutesToTimeSpan(sngEndRT));
}
else
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
}
else
{
// Keep .AcqTimeEnd as the file modification date
// Set .AcqTimeStart based on .AcqEndTime
if (sngEndRT > 0)
{
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd.Subtract(MinutesToTimeSpan(sngEndRT));
}
else
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
}
}
else
{
if (dtNewStartDate >= MINIMUM_ACCEPTABLE_ACQ_START_TIME)
{
datasetFileInfo.AcqTimeStart = dtNewStartDate;
}
}
}
else
{
// Error getting the header info using clsMassLynxNativeIO
// Continue anyway since we've populated some of the values
}
return(true);
} catch (Exception) {
return(false);
}
}
private bool ProcessChemstationMSDataFile(string strDatafilePath, clsDatasetFileInfo datasetFileInfo)
{
bool blnSuccess;
var intCurrentIndex = 0;
try {
using (var oReader = new ChemstationMSFileReader.clsChemstationDataMSFileReader(strDatafilePath)) {
datasetFileInfo.AcqTimeStart = oReader.Header.AcqDate;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(oReader.Header.RetentionTimeMinutesEnd);
datasetFileInfo.ScanCount = oReader.Header.SpectraCount;
for (var intSpectrumIndex = 0; intSpectrumIndex <= datasetFileInfo.ScanCount - 1; intSpectrumIndex++)
{
intCurrentIndex = intSpectrumIndex;
ChemstationMSFileReader.clsSpectralRecord oSpectrum = null;
List <float> lstMZs = null;
List <Int32> lstIntensities = null;
const int intMSLevel = 1;
bool blnValidSpectrum;
try {
oReader.GetSpectrum(intSpectrumIndex, ref oSpectrum);
lstMZs = oSpectrum.Mzs;
lstIntensities = oSpectrum.Intensities;
blnValidSpectrum = true;
} catch (Exception ex) {
OnWarningEvent("Exception obtaining data from the MS file for spectrum index " + intCurrentIndex + ": " + ex.Message);
blnValidSpectrum = false;
}
if (blnValidSpectrum)
{
var objScanStatsEntry = new clsScanStatsEntry
{
ScanNumber = intSpectrumIndex + 1,
ScanType = intMSLevel,
ScanTypeName = "GC-MS",
ScanFilterText = "",
ElutionTime = oSpectrum.RetentionTimeMinutes.ToString("0.0###"),
TotalIonIntensity = StringUtilities.ValueToString(oSpectrum.TIC, 1),
BasePeakIntensity = StringUtilities.ValueToString(oSpectrum.BasePeakAbundance, 1),
BasePeakMZ = oSpectrum.BasePeakMZ.ToString("0.0###"),
BasePeakSignalToNoiseRatio = "0",
IonCount = lstMZs.Count
};
objScanStatsEntry.IonCountRaw = objScanStatsEntry.IonCount;
mDatasetStatsSummarizer.AddDatasetScan(objScanStatsEntry);
if (mSaveTICAndBPI)
{
mTICandBPIPlot.AddData(objScanStatsEntry.ScanNumber, intMSLevel, oSpectrum.RetentionTimeMinutes, oSpectrum.BasePeakAbundance, oSpectrum.TIC);
if (lstMZs.Count > 0)
{
var dblIonsMZ = new double[lstMZs.Count];
var dblIonsIntensity = new double[lstMZs.Count];
for (var intIndex = 0; intIndex <= lstMZs.Count - 1; intIndex++)
{
dblIonsMZ[intIndex] = lstMZs[intIndex];
dblIonsIntensity[intIndex] = lstIntensities[intIndex];
}
mLCMS2DPlot.AddScan(objScanStatsEntry.ScanNumber, intMSLevel, oSpectrum.RetentionTimeMinutes, dblIonsMZ.Length, dblIonsMZ, dblIonsIntensity);
}
}
if (mCheckCentroidingStatus)
{
var lstMzDoubles = new List <double>(lstMZs.Count);
lstMzDoubles.AddRange(lstMZs.Select(ion => (double)ion));
mDatasetStatsSummarizer.ClassifySpectrum(lstMzDoubles, intMSLevel);
}
}
}
}
blnSuccess = true;
} catch (Exception ex) {
// Exception reading file
OnWarningEvent("Exception reading data from the MS file at spectrum index " + intCurrentIndex + ": " + ex.Message);
blnSuccess = false;
}
return(blnSuccess);
}
public abstract bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo);
/// <summary>
/// Reads the MSTS.xml file to determine the acquisition length and the number of scans
/// </summary>
/// <param name="strFolderPath"></param>
/// <param name="datasetFileInfo"></param>
/// <param name="dblTotalAcqTimeMinutes"></param>
/// <returns></returns>
/// <remarks></remarks>
private bool ProcessTimeSegmentFile(string strFolderPath, clsDatasetFileInfo datasetFileInfo, out double dblTotalAcqTimeMinutes)
{
var blnSuccess = false;
double dblStartTime = 0;
double dblEndTime = 0;
dblTotalAcqTimeMinutes = 0;
try
{
datasetFileInfo.ScanCount = 0;
// Open the Contents.xml file
var strFilePath = Path.Combine(strFolderPath, AGILENT_TIME_SEGMENT_FILE);
using (var srReader = new System.Xml.XmlTextReader(new FileStream(strFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
while (!srReader.EOF)
{
srReader.Read();
switch (srReader.NodeType)
{
case System.Xml.XmlNodeType.Element:
switch (srReader.Name)
{
case "TimeSegment":
dblStartTime = 0;
dblEndTime = 0;
break;
case "StartTime":
dblStartTime = srReader.ReadElementContentAsDouble();
break;
case "EndTime":
dblEndTime = srReader.ReadElementContentAsDouble();
break;
case "NumOfScans":
datasetFileInfo.ScanCount += srReader.ReadElementContentAsInt();
blnSuccess = true;
break;
default:
break;
// Ignore it
}
break;
case System.Xml.XmlNodeType.EndElement:
if (srReader.Name == "TimeSegment")
{
// Store the acqtime for this time segment
if (dblEndTime > dblStartTime)
{
blnSuccess = true;
dblTotalAcqTimeMinutes += (dblEndTime - dblStartTime);
}
}
break;
}
}
}
}
catch (Exception ex)
{
// Exception reading file
OnErrorEvent("Exception reading " + AGILENT_TIME_SEGMENT_FILE + ": " + ex.Message, ex);
blnSuccess = false;
}
return(blnSuccess);
}
private void ProcessWiffFile(FileSystemInfo fiDatasetFile, clsDatasetFileInfo datasetFileInfo)
{
try
{
// Open the .Wiff file using the ProteoWizardWrapper
var objPWiz = new pwiz.ProteowizardWrapper.MSDataFileReader(fiDatasetFile.FullName);
try
{
var dtRunStartTime = Convert.ToDateTime(objPWiz.RunStartTime);
// Update AcqTimeEnd if possible
// Found out by trial and error that we need to use .ToUniversalTime() to adjust the time reported by ProteoWizard
dtRunStartTime = dtRunStartTime.ToUniversalTime();
if (dtRunStartTime < datasetFileInfo.AcqTimeEnd)
{
if (datasetFileInfo.AcqTimeEnd.Subtract(dtRunStartTime).TotalDays < 1)
{
datasetFileInfo.AcqTimeStart = dtRunStartTime;
}
}
}
catch (Exception)
{
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd;
}
// Instantiate the Proteowizard Data Parser class
var pWizParser = new clsProteowizardDataParser(objPWiz, mDatasetStatsSummarizer, mTICandBPIPlot,
mLCMS2DPlot, mSaveLCMS2DPlots, mSaveTICAndBPI,
mCheckCentroidingStatus)
{
HighResMS1 = true,
HighResMS2 = true
};
RegisterEvents(pWizParser);
var blnTICStored = false;
var blnSRMDataCached = false;
double dblRuntimeMinutes = 0;
// Note that SRM .Wiff files will only have chromatograms, and no spectra
if (objPWiz.ChromatogramCount > 0)
{
// Process the chromatograms
pWizParser.StoreChromatogramInfo(datasetFileInfo, out blnTICStored, out blnSRMDataCached, out dblRuntimeMinutes);
pWizParser.PossiblyUpdateAcqTimeStart(datasetFileInfo, dblRuntimeMinutes);
}
if (objPWiz.SpectrumCount > 0 & !blnSRMDataCached)
{
// Process the spectral data (though only if we did not process SRM data)
pWizParser.StoreMSSpectraInfo(datasetFileInfo, blnTICStored, ref dblRuntimeMinutes);
pWizParser.PossiblyUpdateAcqTimeStart(datasetFileInfo, dblRuntimeMinutes);
}
objPWiz.Dispose();
clsProgRunner.GarbageCollectNow();
}
catch (Exception ex)
{
OnErrorEvent("Error using ProteoWizard reader: " + ex.Message, ex);
}
}
/// <summary>
/// Process the data file
/// </summary>
/// <param name="strDataFilePath">Dataset folder ptah</param>
/// <param name="datasetFileInfo"></param>
/// <returns></returns>
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
// Returns True if success, False if an error
var blnSuccess = false;
try
{
var diFolder = new DirectoryInfo(strDataFilePath);
datasetFileInfo.FileSystemCreationTime = diFolder.CreationTime;
datasetFileInfo.FileSystemModificationTime = diFolder.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(diFolder.Name);
datasetFileInfo.FileExtension = diFolder.Extension;
// Look for the Analysis.yep file
// Use its modification time to get an initial estimate for the acquisition time
// Assign the .Yep file's size to .FileSizeBytes
var fiYepFile = new FileInfo(Path.Combine(diFolder.FullName, AGILENT_YEP_FILE));
if (fiYepFile.Exists)
{
datasetFileInfo.FileSizeBytes = fiYepFile.Length;
datasetFileInfo.AcqTimeStart = fiYepFile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = fiYepFile.LastWriteTime;
blnSuccess = true;
}
else
{
// Analysis.yep not found; look for Run.log
var fiRunLog = new FileInfo(Path.Combine(diFolder.FullName, AGILENT_RUN_LOG_FILE));
if (fiRunLog.Exists)
{
datasetFileInfo.AcqTimeStart = fiRunLog.LastWriteTime;
datasetFileInfo.AcqTimeEnd = fiRunLog.LastWriteTime;
blnSuccess = true;
// Sum up the sizes of all of the files in this folder
datasetFileInfo.FileSizeBytes = 0;
foreach (var datasetFile in diFolder.GetFiles())
{
datasetFileInfo.FileSizeBytes += datasetFile.Length;
}
}
}
datasetFileInfo.ScanCount = 0;
if (blnSuccess)
{
try
{
// Parse the Run Log file to determine the actual values for .AcqTimeStart and .AcqTimeEnd
ParseRunLogFile(strDataFilePath, datasetFileInfo);
// Parse the Analysis.cdf file to determine the scan count and to further refine .AcqTimeStart
ParseAnalysisCDFFile(strDataFilePath, datasetFileInfo);
}
catch (Exception)
{
// Error parsing the Run Log file or the Analysis.cdf file; do not abort
}
}
}
catch (Exception)
{
blnSuccess = false;
}
return(blnSuccess);
}
private void ParseRunLogFile(string strFolderPath, clsDatasetFileInfo datasetFileInfo)
{
var strMostRecentMethodLine = string.Empty;
try
{
// Try to open the Run.Log file
bool blnProcessedFirstMethodLine;
bool blnEndDateFound;
DateTime dtMethodDate;
using (var srInFile = new StreamReader(Path.Combine(strFolderPath, AGILENT_RUN_LOG_FILE)))
{
blnProcessedFirstMethodLine = false;
blnEndDateFound = false;
while (!srInFile.EndOfStream)
{
var strLineIn = srInFile.ReadLine();
if (string.IsNullOrEmpty(strLineIn))
{
continue;
}
if (!strLineIn.StartsWith(RUN_LOG_FILE_METHOD_LINE_START))
{
continue;
}
strMostRecentMethodLine = string.Copy(strLineIn);
// Method line found
// See if the line contains a key phrase
var intCharLoc = strLineIn.IndexOf(RUN_LOG_FILE_INSTRUMENT_RUNNING, StringComparison.Ordinal);
if (intCharLoc > 0)
{
if (ExtractMethodLineDate(strLineIn, out dtMethodDate))
{
datasetFileInfo.AcqTimeStart = dtMethodDate;
}
blnProcessedFirstMethodLine = true;
}
else
{
intCharLoc = strLineIn.IndexOf(RUN_LOG_INSTRUMENT_RUN_COMPLETED, StringComparison.Ordinal);
if (intCharLoc > 0)
{
if (ExtractMethodLineDate(strLineIn, out dtMethodDate))
{
datasetFileInfo.AcqTimeEnd = dtMethodDate;
blnEndDateFound = true;
}
}
}
// If this is the first method line, then parse out the date and store in .AcqTimeStart
if (!blnProcessedFirstMethodLine)
{
if (ExtractMethodLineDate(strLineIn, out dtMethodDate))
{
datasetFileInfo.AcqTimeStart = dtMethodDate;
}
}
}
}
if (blnProcessedFirstMethodLine & !blnEndDateFound)
{
// Use the last time in the file as the .AcqTimeEnd value
if (ExtractMethodLineDate(strMostRecentMethodLine, out dtMethodDate))
{
datasetFileInfo.AcqTimeEnd = dtMethodDate;
}
}
}
catch (Exception ex)
{
// Run.log file not found
OnWarningEvent("Error in ParseRunLogFile: " + ex.Message);
}
}
public override bool ProcessDataFile(string strDataFilePath, clsDatasetFileInfo datasetFileInfo)
{
// Process a Bruker Xmass folder, specified by strDataFilePath (which can either point to the dataset folder containing the XMass files, or any of the Zip files in the dataset folder)
bool blnSuccess;
try
{
// Determine whether strDataFilePath points to a file or a folder
var diDatasetFolder = GetDatasetFolder(strDataFilePath);
// Validate that we have selected a valid folder
if (!diDatasetFolder.Exists)
{
OnErrorEvent("File/folder not found: " + strDataFilePath);
return(false);
}
// In case we cannot find any .Zip files, update the .AcqTime values to the folder creation date
datasetFileInfo.AcqTimeStart = diDatasetFolder.CreationTime;
datasetFileInfo.AcqTimeEnd = diDatasetFolder.CreationTime;
// Look for the 0_R*.zip files
// If we cannot find any zip files, return false
var lstFiles = diDatasetFolder.GetFiles(ZIPPED_IMAGING_FILE_SEARCH_SPEC).ToList();
if (lstFiles.Count == 0)
{
// 0_R*.zip files not found
OnErrorEvent(ZIPPED_IMAGING_FILE_SEARCH_SPEC + "files not found in " + diDatasetFolder.FullName);
blnSuccess = false;
}
else
{
var fiFirstImagingFile = lstFiles.First();
// Initialize the .DatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.FileSystemCreationTime = fiFirstImagingFile.CreationTime;
mDatasetStatsSummarizer.DatasetFileInfo.FileSystemModificationTime = fiFirstImagingFile.LastWriteTime;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = mDatasetStatsSummarizer.DatasetFileInfo.FileSystemModificationTime;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = mDatasetStatsSummarizer.DatasetFileInfo.FileSystemModificationTime;
mDatasetStatsSummarizer.DatasetFileInfo.DatasetID = datasetFileInfo.DatasetID;
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = diDatasetFolder.Name;
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = fiFirstImagingFile.Extension;
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = 0;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = 0;
// Update the dataset name and file extension
datasetFileInfo.DatasetName = GetDatasetNameViaPath(diDatasetFolder.FullName);
datasetFileInfo.FileExtension = string.Empty;
datasetFileInfo.AcqTimeEnd = DateTime.MinValue;
datasetFileInfo.AcqTimeStart = DateTime.MaxValue;
datasetFileInfo.ScanCount = 0;
// Process each zip file
foreach (var fiFileInfo in lstFiles)
{
// Examine all of the apexAcquisition.method files in this zip file
DetermineAcqStartEndTime(fiFileInfo, datasetFileInfo);
}
if (datasetFileInfo.AcqTimeEnd == DateTime.MinValue || datasetFileInfo.AcqTimeStart == DateTime.MaxValue)
{
// Did not find any apexAcquisition.method files or submethods.xml files
// Use the file modification date of the first zip file
datasetFileInfo.AcqTimeStart = fiFirstImagingFile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = fiFirstImagingFile.LastWriteTime;
}
// Copy over the updated filetime info and scan info from datasetFileInfo to mDatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
blnSuccess = true;
}
}
catch (Exception ex)
{
OnErrorEvent("Exception processing Zipped Imaging Files: " + ex.Message);
blnSuccess = false;
}
return(blnSuccess);
}
