private void AddThermoDevices(
XRawFileIO xcaliburAccessor,
DatasetFileInfo datasetFileInfo,
ICollection <Device> deviceMatchList,
ICollection <Device> deviceSkipList)
{
// ReSharper disable once ForeachCanBePartlyConvertedToQueryUsingAnotherGetEnumerator
foreach (var device in xcaliburAccessor.FileInfo.Devices)
{
if (deviceMatchList.Count > 0 && !deviceMatchList.Contains(device.Key))
{
continue;
}
if (deviceSkipList.Count > 0 && deviceSkipList.Contains(device.Key))
{
continue;
}
for (var deviceNumber = 1; deviceNumber <= device.Value; deviceNumber++)
{
var deviceInfo = xcaliburAccessor.GetDeviceInfo(device.Key, deviceNumber);
datasetFileInfo.DeviceList.Add(deviceInfo);
}
}
}
private bool ParseGCIniFile(string directoryPath, DatasetFileInfo datasetFileInfo)
{
double totalRuntime = 0;
var success = false;
try
{
// Open the GC.ini file
var filePath = Path.Combine(directoryPath, AGILENT_GC_INI_FILE);
if (!File.Exists(filePath))
{
return(false);
}
using (var reader = new StreamReader(filePath))
{
while (!reader.EndOfStream)
{
var dataLine = reader.ReadLine();
if (string.IsNullOrWhiteSpace(dataLine))
{
continue;
}
if (!dataLine.StartsWith("gc.runlength"))
{
continue;
}
// Runtime is the value after the equals sign
var splitLine = dataLine.Split('=');
if (splitLine.Length <= 1)
{
continue;
}
if (double.TryParse(splitLine[1], out totalRuntime))
{
success = true;
}
}
}
}
catch (Exception ex)
{
// Exception reading file
OnErrorEvent("Exception reading " + AGILENT_GC_INI_FILE + ": " + ex.Message, ex);
success = false;
}
if (success)
{
// Update the acquisition start time
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd.AddMinutes(-totalRuntime);
}
return(success);
}
/// <summary>
/// This function is used to determine one or more overall quality scores
/// </summary>
/// <param name="xcaliburAccessor"></param>
/// <param name="datasetFileInfo"></param>
/// <remarks></remarks>
private void ComputeQualityScores(XRawFileIO xcaliburAccessor, DatasetFileInfo datasetFileInfo)
{
float overallScore;
double overallAvgIntensitySum = 0;
var overallAvgCount = 0;
if (mLCMS2DPlot.ScanCountCached > 0)
{
// Obtain the overall average intensity value using the data cached in mLCMS2DPlot
// This avoids having to reload all of the data using xcaliburAccessor
const int msLevelFilter = 1;
overallScore = mLCMS2DPlot.ComputeAverageIntensityAllScans(msLevelFilter);
}
else
{
var scanCount = xcaliburAccessor.GetNumScans();
GetStartAndEndScans(scanCount, out var scanStart, out var scanEnd);
for (var scanNumber = scanStart; scanNumber <= scanEnd; scanNumber++)
{
// This function returns the number of points in massIntensityPairs()
var returnCode = xcaliburAccessor.GetScanData2D(scanNumber, out var massIntensityPairs);
if (returnCode <= 0)
{
continue;
}
if (massIntensityPairs == null || massIntensityPairs.GetLength(1) <= 0)
{
continue;
}
// Keep track of the quality scores and then store one or more overall quality scores in datasetFileInfo.OverallQualityScore
// For now, this just computes the average intensity for each scan and then computes and overall average intensity value
double intensitySum = 0;
for (var ionIndex = 0; ionIndex <= massIntensityPairs.GetUpperBound(1); ionIndex++)
{
intensitySum += massIntensityPairs[1, ionIndex];
}
overallAvgIntensitySum += intensitySum / massIntensityPairs.GetLength(1);
overallAvgCount += 1;
}
if (overallAvgCount > 0)
{
overallScore = (float)(overallAvgIntensitySum / overallAvgCount);
}
else
{
overallScore = 0;
}
}
datasetFileInfo.OverallQualityScore = overallScore;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="masicOptions"></param>
/// <param name="peakFinder"></param>
/// <param name="parentIonProcessor"></param>
protected clsDataImport(
clsMASICOptions masicOptions,
MASICPeakFinder.clsMASICPeakFinder peakFinder,
clsParentIonProcessing parentIonProcessor,
clsScanTracking scanTracking)
{
mOptions = masicOptions;
mPeakFinder = peakFinder;
mParentIonProcessor = parentIonProcessor;
mScanTracking = scanTracking;
mDatasetFileInfo = new DatasetFileInfo();
mInterferenceCalculator = new InterDetect.InterferenceCalculator();
mInterferenceCalculator.StatusEvent += OnStatusEvent;
mInterferenceCalculator.ErrorEvent += OnErrorEvent;
mInterferenceCalculator.WarningEvent += InterferenceWarningEventHandler;
mCachedPrecursorIons = new List <InterDetect.Peak>();
mCachedPrecursorScan = 0;
mIsolationWidthNotFoundCount = 0;
mPrecursorNotFoundCount = 0;
}
/** @return A single GetDatasetWriteTicketsInputData for uploading ReadMe.txt. */
private GetDatasetWriteTicketsInputData getSingleGetDatasetWriteTicketsInputData(DataManagementService dmService)
{
// Create a Dataset
DatasetProperties2 props = new DatasetProperties2();
props.ClientId = "datasetWriteTixTestClientId";
props.Type = "Text";
props.Name = "Sample-FMS-Upload";
props.Description = "Testing put File";
DatasetProperties2[] currProps = { props };
CreateDatasetsResponse resp = dmService.CreateDatasets2(currProps);
// Assume this file is in current dir
FileInfo file1 = new FileInfo("ReadMe.txt");
// Create a file to associate with dataset
DatasetFileInfo fileInfo = new DatasetFileInfo();
fileInfo.ClientId = "file_1";
fileInfo.FileName = file1.FullName;
fileInfo.NamedReferencedName = "Text";
fileInfo.IsText = true;
fileInfo.AllowReplace = false;
DatasetFileInfo[] fileInfos = { fileInfo };
GetDatasetWriteTicketsInputData inputData = new GetDatasetWriteTicketsInputData();
inputData.Dataset = resp.Output[0].Dataset;
inputData.CreateNewVersion = false;
inputData.DatasetFileInfos = fileInfos;
return(inputData);
}
public void PossiblyUpdateAcqTimeStart(DatasetFileInfo datasetFileInfo, double runtimeMinutes)
{
if (runtimeMinutes > 0)
{
var acqTimeStartAlt = datasetFileInfo.AcqTimeEnd.AddMinutes(-runtimeMinutes);
if (acqTimeStartAlt < datasetFileInfo.AcqTimeStart && datasetFileInfo.AcqTimeStart.Subtract(acqTimeStartAlt).TotalDays < 1)
{
datasetFileInfo.AcqTimeStart = acqTimeStartAlt;
}
}
}
private void ProcessRawDirectory(DirectoryInfo datasetDirectory, DatasetFileInfo datasetFileInfo, out List <FileInfo> primaryDataFiles)
{
primaryDataFiles = new List <FileInfo>();
// Sum up the sizes of all of the files in this directory
datasetFileInfo.FileSizeBytes = 0;
var fileCount = 0;
foreach (var item in datasetDirectory.GetFiles())
{
datasetFileInfo.FileSizeBytes += item.Length;
if (fileCount == 0)
{
// Assign the first file's modification time to .AcqTimeStart and .AcqTimeEnd
// Necessary in case _header.txt is missing
datasetFileInfo.AcqTimeStart = item.LastWriteTime;
datasetFileInfo.AcqTimeEnd = item.LastWriteTime;
}
if (item.Name.ToLower() == "_header.txt")
{
// Assign the file's modification time to .AcqTimeStart and .AcqTimeEnd
// These will get updated below to more precise values
datasetFileInfo.AcqTimeStart = item.LastWriteTime;
datasetFileInfo.AcqTimeEnd = item.LastWriteTime;
}
if (item.Extension.ToLower().Equals(".dat"))
{
primaryDataFiles.Add(item);
}
fileCount += 1;
}
var nativeFileIO = new clsMassLynxNativeIO();
if (nativeFileIO.GetFileInfo(datasetDirectory.FullName, out var headerInfo))
{
ReadMassLynxAcquisitionInfo(datasetDirectory, datasetFileInfo, nativeFileIO, headerInfo);
}
else
{
// Error getting the header info using clsMassLynxNativeIO
// Continue anyway since we've populated some of the values
}
LoadScanDataWithProteoWizard(datasetDirectory, datasetFileInfo, true);
}
private bool ParseICRDirectory(DirectoryInfo icrDirectory, DatasetFileInfo datasetFileInfo)
{
// Look for and open the .Pek file in icrDirectory
// Count the number of PEK_FILE_FILENAME_LINE lines
var fileListCount = 0;
var success = false;
foreach (var pekFile in icrDirectory.GetFiles("*.pek"))
{
try
{
// Try to open the PEK file
fileListCount = 0;
using (var reader = new StreamReader(pekFile.OpenRead()))
{
while (!reader.EndOfStream)
{
var dataLine = reader.ReadLine();
if (string.IsNullOrWhiteSpace(dataLine))
{
continue;
}
if (dataLine.StartsWith(PEK_FILE_FILENAME_LINE))
{
fileListCount += 1;
}
}
}
success = true;
}
catch (Exception)
{
// Error opening or parsing the PEK file
success = false;
}
if (fileListCount > datasetFileInfo.ScanCount)
{
datasetFileInfo.ScanCount = fileListCount;
}
// Only parse the first .Pek file found
break;
}
return(success);
}
/// <summary>
/// Process the DeconTools results
/// </summary>
/// <param name="dataFilePath">Isos file path</param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error</returns>
/// <remarks>Will also read the _scans.csv file if present (to determine ElutionTime and MSLevel</remarks>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
ResetResults();
var isosFile = new FileInfo(dataFilePath);
if (!isosFile.Exists)
{
OnErrorEvent("_isos.csv file not found: " + dataFilePath);
return(false);
}
var datasetID = DatasetID;
// Record the file size and Dataset ID
datasetFileInfo.FileSystemCreationTime = isosFile.CreationTime;
datasetFileInfo.FileSystemModificationTime = isosFile.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = datasetID;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(isosFile.Name);
datasetFileInfo.FileExtension = isosFile.Extension;
datasetFileInfo.FileSizeBytes = isosFile.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
if (mSaveTICAndBPI || mCreateDatasetInfoFile || mCreateScanStatsFile || mSaveLCMS2DPlots)
{
// Load data from each scan
// This is used to create the TIC and BPI plot, the 2D LC/MS plot, and/or to create the Dataset Info File
LoadData(isosFile, datasetFileInfo);
}
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
// This will also compute the SHA-1 hash of the isos file and add it to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetFileStats(isosFile, datasetID);
// Copy over the updated file time info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetStatsSummarizerUsingDatasetFileInfo(datasetFileInfo);
PostProcessTasks();
return(true);
}
/**
* @return An array of NUMBER_OF_DATASETS GetDatasetWriteTicketsInputData objects
* for uploading NUMBER_OF_FILES_PER_DATASET copies of ReadMe.txt to each Dataset.
*/
private GetDatasetWriteTicketsInputData[] getMultipleGetDatasetWriteTicketsInputData(DataManagementService dmService)
{
GetDatasetWriteTicketsInputData[] inputs = new GetDatasetWriteTicketsInputData[NUMBER_OF_DATASETS];
DatasetProperties2[] currProps = new DatasetProperties2[inputs.Length];
// Create a bunch of Datasets
for (int i = 0; i < inputs.Length; ++i)
{
DatasetProperties2 props = new DatasetProperties2();
props.ClientId = "datasetWriteTixTestClientId " + i;
props.Type = "Text";
props.Name = "Sample-FMS-Upload-" + i;
props.Description = "Testing Multiple put File";
currProps[i] = props;
}
CreateDatasetsResponse resp = dmService.CreateDatasets2(currProps);
// Create files to associate with each Dataset
for (int i = 0; i < inputs.Length; ++i)
{
DatasetFileInfo[] fileInfos = new DatasetFileInfo[NUMBER_OF_FILES_PER_DATASET];
for (int j = 0; j < fileInfos.Length; ++j)
{
DatasetFileInfo fileInfo = new DatasetFileInfo();
// Create different filenames to be uploaded into the same dataset
// Create or use this file in current dir
FileInfo file1 = new FileInfo("ReadMeCopy" + j + ".txt");
assureFileCreated(file1);
fileInfo.ClientId = "Dataset " + i + " File " + j;
fileInfo.FileName = file1.FullName;
fileInfo.NamedReferencedName = "Text";
fileInfo.IsText = true;
fileInfo.AllowReplace = false;
fileInfos[j] = fileInfo;
}
GetDatasetWriteTicketsInputData inputData = new GetDatasetWriteTicketsInputData();
inputData.Dataset = resp.Output[i].Dataset;
inputData.CreateNewVersion = false;
inputData.DatasetFileInfos = fileInfos;
inputs[i] = inputData;
}
return(inputs);
}
private bool ParseBrukerAcquFile(string directoryPath, DatasetFileInfo datasetFileInfo)
{
bool success;
try
{
// Try to open the acqu file
success = false;
using (var reader = new StreamReader(Path.Combine(directoryPath, BRUKER_ACQU_FILE)))
{
while (!reader.EndOfStream)
{
var dataLine = reader.ReadLine();
if (string.IsNullOrWhiteSpace(dataLine))
{
continue;
}
if (!dataLine.StartsWith(BRUKER_ACQU_FILE_ACQ_LINE_START))
{
continue;
}
// Date line found
// It is of the form: ##$AQ_DATE= <Sat Aug 20 07:56:55 2005>
dataLine = dataLine.Substring(BRUKER_ACQU_FILE_ACQ_LINE_START.Length).Trim();
dataLine = dataLine.TrimEnd(BRUKER_ACQU_FILE_ACQ_LINE_END);
success = ParseBrukerDateFromArray(dataLine, out var acqDate);
if (success)
{
datasetFileInfo.AcqTimeEnd = acqDate;
}
break;
}
}
}
catch (Exception)
{
// Error opening the acqu file
success = false;
}
return(success);
}
/// <summary>
/// Process the dataset
/// </summary>
/// <param name="dataFilePath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error</returns>
/// <remarks></remarks>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
ResetResults();
// Override dataFilePath here, if needed
// dataFilePath = dataFilePath;
// Obtain the full path to the file
var datasetFile = new FileInfo(dataFilePath);
datasetFileInfo.FileSystemCreationTime = datasetFile.CreationTime;
datasetFileInfo.FileSystemModificationTime = datasetFile.LastWriteTime;
// Using the file system modification time as the acquisition end time
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = 0;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(datasetFile.Name);
datasetFileInfo.FileExtension = datasetFile.Extension;
datasetFileInfo.FileSizeBytes = datasetFile.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
mLCMS2DPlot.Options.UseObservedMinScan = false;
LoadScanDataWithProteoWizard(datasetFile, datasetFileInfo, true);
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
// This will also compute the SHA-1 hash of the .Wiff file and add it to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetFileStats(datasetFile, datasetFileInfo.DatasetID);
// Copy over the updated file time info and scan info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
PostProcessTasks();
return(true);
}
/// <summary>
/// Process the dataset
/// </summary>
/// <param name="dataFilePath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error or if the file has no scans</returns>
/// <remarks></remarks>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
ResetResults();
// Obtain the full path to the file
var instrumentDataFile = new FileInfo(dataFilePath);
if (!instrumentDataFile.Exists)
{
OnErrorEvent("Instrument file not found: " + dataFilePath);
return(false);
}
// Future, optional: Determine the DatasetID
// Unfortunately, this is not present in metadata.txt
// datasetID = LookupDatasetID(datasetName)
var datasetID = DatasetID;
// Record the file size and Dataset ID
datasetFileInfo.FileSystemCreationTime = instrumentDataFile.CreationTime;
datasetFileInfo.FileSystemModificationTime = instrumentDataFile.LastWriteTime;
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = datasetID;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(instrumentDataFile.Name);
datasetFileInfo.FileExtension = instrumentDataFile.Extension;
datasetFileInfo.FileSizeBytes = instrumentDataFile.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
// Read the file info from the file system
// This will also compute the SHA-1 hash of the .Raw file and add it to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetFileStats(instrumentDataFile, datasetID);
// Copy over the updated file time info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetStatsSummarizerUsingDatasetFileInfo(datasetFileInfo);
PostProcessTasks();
return(true);
}
/// <summary>
/// Process the dataset
/// </summary>
/// <param name="dataFilePath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error</returns>
/// <remarks></remarks>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
ResetResults();
try
{
var datasetDirectory = new DirectoryInfo(dataFilePath);
datasetFileInfo.FileSystemCreationTime = datasetDirectory.CreationTime;
datasetFileInfo.FileSystemModificationTime = datasetDirectory.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(datasetDirectory.Name);
datasetFileInfo.FileExtension = datasetDirectory.Extension;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
mLCMS2DPlot.Options.UseObservedMinScan = false;
ProcessRawDirectory(datasetDirectory, datasetFileInfo, out var primaryDataFiles);
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
// This will also add the primary data files to mDatasetStatsSummarizer.DatasetFileInfo
// The SHA-1 hash of the first file in primaryDataFiles will also be computed
UpdateDatasetFileStats(datasetDirectory, primaryDataFiles, datasetFileInfo.DatasetID);
// Copy over the updated file time info and scan info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetStatsSummarizerUsingDatasetFileInfo(datasetFileInfo);
PostProcessTasks();
return(true);
}
catch (Exception)
{
PostProcessTasks();
return(false);
}
}
private void ParseAnalysisCDFFile(string directoryPath, DatasetFileInfo datasetFileInfo)
{
NetCDFReader.clsMSNetCdf netCDFReader = null;
try
{
// Note: as of May 2016 this only works if you compile as x86 or if you enable "Prefer 32-bit" when compiling as AnyCPU
// In contrast, XRawFileIO in clsThermoRawFileInfoScanner requires that "Prefer 32-bit" be disabled
netCDFReader = new NetCDFReader.clsMSNetCdf();
var success = netCDFReader.OpenMSCdfFile(Path.Combine(directoryPath, AGILENT_ANALYSIS_CDF_FILE));
if (success)
{
var scanCount = netCDFReader.GetScanCount();
if (scanCount > 0)
{
// Lookup the scan time of the final scan
if (netCDFReader.GetScanInfo(scanCount - 1, out var scanNumber,
out _, out var scanTime, out _, out _))
{
// Add 1 to scanNumber since the scan number is off by one in the CDF file
datasetFileInfo.ScanCount = scanNumber + 1;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.Add(SecondsToTimeSpan(scanTime));
}
}
else
{
datasetFileInfo.ScanCount = 0;
}
}
}
catch (Exception ex)
{
OnWarningEvent("Error in ParseAnalysisCDFFile: " + ex.Message);
}
finally
{
netCDFReader?.CloseMSCdfFile();
}
}
private bool ParseBrukerLockFile(string directoryPath, DatasetFileInfo datasetFileInfo)
{
bool success;
try
{
// Try to open the Lock file
// The date line is the first (and only) line in the file
success = false;
using (var reader = new StreamReader(Path.Combine(directoryPath, BRUKER_LOCK_FILE)))
{
if (!reader.EndOfStream)
{
var dataLine = reader.ReadLine();
if (dataLine != null)
{
// Date line found
// It is of the form: wd37119 2208 WD37119\9TOperator Sat Aug 20 06:10:31 2005
success = ParseBrukerDateFromArray(dataLine, out var acqDate);
if (success)
{
datasetFileInfo.AcqTimeStart = acqDate;
}
}
}
}
}
catch (Exception)
{
// Error opening the Lock file
success = false;
}
return(success);
}
public abstract bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo);
private void ParseRunLogFile(string directoryPath, DatasetFileInfo datasetFileInfo)
{
var mostRecentMethodLine = string.Empty;
try
{
// Try to open the Run.Log file
bool processedFirstMethodLine;
bool endDateFound;
DateTime methodDate;
using (var reader = new StreamReader(Path.Combine(directoryPath, AGILENT_RUN_LOG_FILE)))
{
processedFirstMethodLine = false;
endDateFound = false;
while (!reader.EndOfStream)
{
var dataLine = reader.ReadLine();
if (string.IsNullOrWhiteSpace(dataLine))
{
continue;
}
if (!dataLine.StartsWith(RUN_LOG_FILE_METHOD_LINE_START))
{
continue;
}
mostRecentMethodLine = string.Copy(dataLine);
// Method line found
// See if the line contains a key phrase
var charIndex = dataLine.IndexOf(RUN_LOG_FILE_INSTRUMENT_RUNNING, StringComparison.Ordinal);
if (charIndex > 0)
{
if (ExtractMethodLineDate(dataLine, out methodDate))
{
datasetFileInfo.AcqTimeStart = methodDate;
}
processedFirstMethodLine = true;
}
else
{
charIndex = dataLine.IndexOf(RUN_LOG_INSTRUMENT_RUN_COMPLETED, StringComparison.Ordinal);
if (charIndex > 0)
{
if (ExtractMethodLineDate(dataLine, out methodDate))
{
datasetFileInfo.AcqTimeEnd = methodDate;
endDateFound = true;
}
}
}
// If this is the first method line, then parse out the date and store in .AcqTimeStart
if (!processedFirstMethodLine)
{
if (ExtractMethodLineDate(dataLine, out methodDate))
{
datasetFileInfo.AcqTimeStart = methodDate;
}
}
}
}
if (processedFirstMethodLine && !endDateFound)
{
// Use the last time in the file as the .AcqTimeEnd value
if (ExtractMethodLineDate(mostRecentMethodLine, out methodDate))
{
datasetFileInfo.AcqTimeEnd = methodDate;
}
}
}
catch (Exception ex)
{
// Run.log file not found
OnWarningEvent("Error in ParseRunLogFile: " + ex.Message);
}
}
/// <summary>
/// Process the data file
/// </summary>
/// <param name="dataFilePath">Dataset directory ptah</param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error</returns>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
var success = false;
ResetResults();
try
{
var datasetDirectory = new DirectoryInfo(dataFilePath);
datasetFileInfo.FileSystemCreationTime = datasetDirectory.CreationTime;
datasetFileInfo.FileSystemModificationTime = datasetDirectory.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(datasetDirectory.Name);
datasetFileInfo.FileExtension = datasetDirectory.Extension;
// Look for the Analysis.yep file
// Use its modification time to get an initial estimate for the acquisition time
// Assign the .Yep file's size to .FileSizeBytes
var yepFile = new FileInfo(Path.Combine(datasetDirectory.FullName, AGILENT_YEP_FILE));
if (yepFile.Exists)
{
datasetFileInfo.FileSizeBytes = yepFile.Length;
datasetFileInfo.AcqTimeStart = yepFile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = yepFile.LastWriteTime;
if (mDisableInstrumentHash)
{
mDatasetStatsSummarizer.DatasetFileInfo.AddInstrumentFileNoHash(yepFile);
}
else
{
mDatasetStatsSummarizer.DatasetFileInfo.AddInstrumentFile(yepFile);
}
success = true;
}
else
{
// Analysis.yep not found; look for Run.log
var runLog = new FileInfo(Path.Combine(datasetDirectory.FullName, AGILENT_RUN_LOG_FILE));
if (runLog.Exists)
{
datasetFileInfo.AcqTimeStart = runLog.LastWriteTime;
datasetFileInfo.AcqTimeEnd = runLog.LastWriteTime;
success = true;
// Sum up the sizes of all of the files in this directory
datasetFileInfo.FileSizeBytes = 0;
foreach (var datasetFile in datasetDirectory.GetFiles())
{
datasetFileInfo.FileSizeBytes += datasetFile.Length;
}
// Compute the SHA-1 hash of the largest file in instrument directory
AddLargestInstrumentFile(datasetDirectory);
}
}
datasetFileInfo.ScanCount = 0;
if (success)
{
try
{
// Parse the Run Log file to determine the actual values for .AcqTimeStart and .AcqTimeEnd
ParseRunLogFile(dataFilePath, datasetFileInfo);
// Parse the Analysis.cdf file to determine the scan count and to further refine .AcqTimeStart
ParseAnalysisCDFFile(dataFilePath, datasetFileInfo);
}
catch (Exception)
{
// Error parsing the Run Log file or the Analysis.cdf file; do not abort
}
}
// Copy over the updated file time info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetStatsSummarizerUsingDatasetFileInfo(datasetFileInfo);
PostProcessTasks();
}
catch (Exception)
{
success = false;
}
PostProcessTasks();
return(success);
}
/// <summary>
/// Reads the Contents.xml file to look for the AcquiredTime entry
/// </summary>
/// <param name="directoryPath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if the file exists and the AcquiredTime entry was successfully parsed; otherwise false</returns>
/// <remarks></remarks>
private bool ProcessContentsXMLFile(string directoryPath, DatasetFileInfo datasetFileInfo)
{
var success = false;
try
{
// Open the Contents.xml file
var filePath = Path.Combine(directoryPath, AGILENT_XML_CONTENTS_FILE);
using (var reader = new System.Xml.XmlTextReader(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
while (!reader.EOF)
{
reader.Read();
switch (reader.NodeType)
{
case System.Xml.XmlNodeType.Element:
if (reader.Name == "AcquiredTime")
{
try
{
var acquisitionStartTime = reader.ReadElementContentAsDateTime();
// Convert from Universal time to Local time
var acquisitionTime = acquisitionStartTime.ToLocalTime();
// ReSharper disable CommentTypo
// There have been some cases where the acquisition start time is several years before the file modification time,
// for example XG_A83CapiHSSWash1.d where the time in the Contents.xml file is 3/20/2005 while the file modification time is 2010
// Thus, we use a sanity check of a maximum run time of 24 hours
// ReSharper restore CommentTypo
if (datasetFileInfo.AcqTimeEnd.Subtract(acquisitionTime).TotalDays < 1)
{
datasetFileInfo.AcqTimeStart = acquisitionStartTime.ToLocalTime();
success = true;
}
}
catch (Exception)
{
// Ignore errors here
}
}
break;
}
}
}
}
catch (Exception ex)
{
// Exception reading file
OnErrorEvent("Exception reading " + AGILENT_XML_CONTENTS_FILE + ": " + ex.Message, ex);
success = false;
}
return(success);
}
private void ReadBinaryData(string dataDirectoryPath, DatasetFileInfo datasetFileInfo, double acquisitionLengthMinutes)
{
try
{
// Open the data directory using the ProteoWizardWrapper
var pWiz = new pwiz.ProteowizardWrapper.MSDataFileReader(dataDirectoryPath);
try
{
var runStartTime = Convert.ToDateTime(pWiz.RunStartTime);
// Update AcqTimeEnd if possible
if (runStartTime < datasetFileInfo.AcqTimeEnd)
{
if (datasetFileInfo.AcqTimeEnd.Subtract(runStartTime).TotalDays < 1)
{
datasetFileInfo.AcqTimeStart = runStartTime;
if (acquisitionLengthMinutes > 0)
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(acquisitionLengthMinutes);
}
}
}
}
catch (Exception)
{
// Leave the times unchanged
}
// Instantiate the ProteoWizard Data Parser class
var pWizParser = new clsProteoWizardDataParser(pWiz, mDatasetStatsSummarizer, mTICAndBPIPlot,
mLCMS2DPlot, mSaveLCMS2DPlots, mSaveTICAndBPI,
mCheckCentroidingStatus)
{
HighResMS1 = true,
HighResMS2 = true
};
RegisterEvents(pWizParser);
var ticStored = false;
double runtimeMinutes = 0;
if (pWiz.ChromatogramCount > 0)
{
// Process the chromatograms
pWizParser.StoreChromatogramInfo(datasetFileInfo, out ticStored, out _, out runtimeMinutes);
pWizParser.PossiblyUpdateAcqTimeStart(datasetFileInfo, runtimeMinutes);
}
if (pWiz.SpectrumCount > 0)
{
// Process the spectral data
var skipExistingScans = (pWiz.ChromatogramCount > 0);
pWizParser.StoreMSSpectraInfo(ticStored, ref runtimeMinutes,
skipExistingScans,
skipScansWithNoIons: true,
maxScansToTrackInDetail: MAX_SCANS_TO_TRACK_IN_DETAIL,
maxScansForTicAndBpi: MAX_SCANS_FOR_TIC_AND_BPI);
pWizParser.PossiblyUpdateAcqTimeStart(datasetFileInfo, runtimeMinutes);
}
pWiz.Dispose();
ProgRunner.GarbageCollectNow();
}
catch (Exception ex)
{
OnErrorEvent("Exception reading the Binary Data in the Agilent TOF .D directory using ProteoWizard: " + ex.Message, ex);
}
}
/// <summary>
/// Process the dataset
/// </summary>
/// <param name="dataFilePath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error</returns>
/// <remarks></remarks>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
var success = false;
ResetResults();
try
{
var rootDirectory = new DirectoryInfo(dataFilePath);
var acqDataDirectory = new DirectoryInfo(Path.Combine(rootDirectory.FullName, AGILENT_ACQDATA_FOLDER_NAME));
datasetFileInfo.FileSystemCreationTime = acqDataDirectory.CreationTime;
datasetFileInfo.FileSystemModificationTime = acqDataDirectory.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(rootDirectory.Name);
datasetFileInfo.FileExtension = rootDirectory.Extension;
datasetFileInfo.FileSizeBytes = 0;
datasetFileInfo.ScanCount = 0;
if (acqDataDirectory.Exists)
{
// Sum up the sizes of all of the files in the AcqData directory
foreach (var file in acqDataDirectory.GetFiles("*", SearchOption.AllDirectories))
{
datasetFileInfo.FileSizeBytes += file.Length;
}
// Look for the MSScan.bin file
// Use its modification time to get an initial estimate for the acquisition end time
var msScanFile = new FileInfo(Path.Combine(acqDataDirectory.FullName, AGILENT_MS_SCAN_FILE));
bool primaryFileAdded;
if (msScanFile.Exists)
{
datasetFileInfo.AcqTimeStart = msScanFile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = msScanFile.LastWriteTime;
// Read the file info from the file system
// Several of these stats will be further updated later
// This will also compute the SHA-1 hash of the MSScan.bin file and add it to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetFileStats(msScanFile, datasetFileInfo.DatasetID, out primaryFileAdded);
}
else
{
// Read the file info from the file system
// Several of these stats will be further updated later
UpdateDatasetFileStats(acqDataDirectory, msScanFile, datasetFileInfo.DatasetID);
primaryFileAdded = false;
}
var additionalFilesToHash = new List <FileInfo>
{
new FileInfo(Path.Combine(acqDataDirectory.FullName, AGILENT_MS_PEAK_FILE)),
new FileInfo(Path.Combine(acqDataDirectory.FullName, AGILENT_MS_PEAK_PERIODIC_ACTUALS_FILE)),
new FileInfo(Path.Combine(acqDataDirectory.FullName, AGILENT_MS_PROFILE_FILE)),
new FileInfo(Path.Combine(acqDataDirectory.FullName, AGILENT_XML_CONTENTS_FILE))
};
foreach (var addnlFile in additionalFilesToHash)
{
if (!addnlFile.Exists)
{
continue;
}
if (mDisableInstrumentHash)
{
mDatasetStatsSummarizer.DatasetFileInfo.AddInstrumentFileNoHash(addnlFile);
}
else
{
mDatasetStatsSummarizer.DatasetFileInfo.AddInstrumentFile(addnlFile);
}
primaryFileAdded = true;
}
if (!primaryFileAdded)
{
// Add largest file in the AcqData directory
AddLargestInstrumentFile(acqDataDirectory);
}
success = true;
}
if (success)
{
// The AcqData directory exists
// Parse the Contents.xml file to determine the acquisition start time
var acqStartTimeDetermined = ProcessContentsXMLFile(acqDataDirectory.FullName, datasetFileInfo);
// Parse the MSTS.xml file to determine the acquisition length and number of scans
var validMSTS = ProcessTimeSegmentFile(acqDataDirectory.FullName, datasetFileInfo, out var acquisitionLengthMinutes);
if (!acqStartTimeDetermined && validMSTS)
{
// Compute the start time from .AcqTimeEnd minus acquisitionLengthMinutes
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd.AddMinutes(-acquisitionLengthMinutes);
}
// Note: could parse the AcqMethod.xml file to determine if MS2 spectra are present
//<AcqMethod>
// <QTOF>
// <TimeSegment>
// <Acquisition>
// <AcqMode>TargetedMS2</AcqMode>
// Read the raw data to create the TIC and BPI
ReadBinaryData(rootDirectory.FullName, datasetFileInfo, acquisitionLengthMinutes);
}
if (success)
{
// Copy over the updated file time info and scan info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
}
}
catch (Exception ex)
{
OnErrorEvent("Exception parsing Agilent TOF .D directory: " + ex.Message, ex);
success = false;
}
PostProcessTasks();
return(success);
}
/// <summary>
/// Reads the MSTS.xml file to determine the acquisition length and the number of scans
/// </summary>
/// <param name="directoryPath"></param>
/// <param name="datasetFileInfo"></param>
/// <param name="totalAcqTimeMinutes"></param>
/// <returns></returns>
/// <remarks></remarks>
private bool ProcessTimeSegmentFile(string directoryPath, DatasetFileInfo datasetFileInfo, out double totalAcqTimeMinutes)
{
var success = false;
double startTime = 0;
double endTime = 0;
totalAcqTimeMinutes = 0;
try
{
datasetFileInfo.ScanCount = 0;
// Open the Contents.xml file
var filePath = Path.Combine(directoryPath, AGILENT_TIME_SEGMENT_FILE);
using (var reader = new System.Xml.XmlTextReader(new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
while (!reader.EOF)
{
reader.Read();
switch (reader.NodeType)
{
case System.Xml.XmlNodeType.Element:
switch (reader.Name)
{
case "TimeSegment":
startTime = 0;
endTime = 0;
break;
case "StartTime":
startTime = reader.ReadElementContentAsDouble();
break;
case "EndTime":
endTime = reader.ReadElementContentAsDouble();
break;
case "NumOfScans":
datasetFileInfo.ScanCount += reader.ReadElementContentAsInt();
success = true;
break;
default:
// Ignore it
break;
}
break;
case System.Xml.XmlNodeType.EndElement:
if (reader.Name == "TimeSegment")
{
// Store the acqtime for this time segment
if (endTime > startTime)
{
success = true;
totalAcqTimeMinutes += endTime - startTime;
}
}
break;
}
}
}
}
catch (Exception ex)
{
// Exception reading file
OnErrorEvent("Exception reading " + AGILENT_TIME_SEGMENT_FILE + ": " + ex.Message, ex);
success = false;
}
return(success);
}
/// <summary>
/// Process the dataset
/// </summary>
/// <param name="dataFilePath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error</returns>
/// <remarks></remarks>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
ResetResults();
DataReader uimfReader = null;
GlobalParams globalParams = null;
var masterFrameNumList = new int[1];
// Obtain the full path to the file
var uimfFile = new FileInfo(dataFilePath);
if (!uimfFile.Exists)
{
return(false);
}
// Future, optional: Determine the DatasetID
// Unfortunately, this is not present in metadata.txt
// datasetID = LookupDatasetID(datasetName)
var datasetID = 0;
datasetFileInfo.FileSystemCreationTime = uimfFile.CreationTime;
datasetFileInfo.FileSystemModificationTime = uimfFile.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = datasetID;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(uimfFile.Name);
datasetFileInfo.FileExtension = uimfFile.Extension;
datasetFileInfo.FileSizeBytes = uimfFile.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
var readError = false;
var inaccurateStartTime = false;
try
{
// Use the UIMFLibrary to read the .UIMF file
uimfReader = new DataReader(uimfFile.FullName);
}
catch (Exception ex)
{
// File open failed
OnErrorEvent("Call to .OpenUIMF failed for " + uimfFile.Name + ": " + ex.Message, ex);
readError = true;
}
if (!readError)
{
try
{
// First obtain the global parameters
globalParams = uimfReader.GetGlobalParams();
}
catch (Exception)
{
// Read error
readError = true;
}
}
if (!readError)
{
// Read the file info
var dctMasterFrameList = new Dictionary <int, UIMFData.FrameType>();
try
{
// Construct a master list of frame numbers and frame types
dctMasterFrameList = uimfReader.GetMasterFrameList();
if (dctMasterFrameList.Count > 0)
{
// Copy the frame numbers into an array so that we can assure it's sorted
masterFrameNumList = new int[dctMasterFrameList.Keys.Count];
dctMasterFrameList.Keys.CopyTo(masterFrameNumList, 0);
Array.Sort(masterFrameNumList);
}
// ReSharper disable CommentTypo
// Extract the acquisition time information
// The Global_Parameters table records the start time of the entire dataset in field DateStarted
// The Frame_Parameters table records the start time of reach frame in field StartTime
// The DateStarted column in the Global_Parameters table should be represented by one of these values
// A text-based date, like "5/2/2011 4:26:59 PM"; example: Sarc_MS2_26_2Apr11_Cheetah_11-02-18_inverse.uimf
// A text-based date (no time info), like "Thursday, January 13, 2011"; example: QC_Shew_11_01_pt5_c2_030311_earth_4ms_0001
// A tick-based date, like 129272890050787740 (number of ticks since January 1, 1601); example: BATs_TS_01_c4_Eagle_10-02-06_0000
// The StartTime column in the Frame_Parameters table should be represented by one of these values
// Integer between 0 and 1440 representing number of minutes since midnight (can loop from 1439.9 to 0); example: Sarc_MS2_26_2Apr11_Cheetah_11-02-18_inverse.uimf
// Integer between 0 and 60 representing number of minutes since past the current hour (can loop from 59.9 to 0); example: BATs_TS_01_c4_Eagle_10-02-06_0000.uimf
// A tick-based date, like 634305349108974800 (number of ticks since January 1, 0001); example: QC_Shew_11_01_pt5_c2_030311_earth_4ms_0001
// A negative number representing number of minutes from the start of the run in UTC time to the start of the current frame, in local time; example: Sarc_P08_G03_0747_7Dec11_Cheetah_11-09-05.uimf
// Examine values: Frame 1 has StartTime = -479.993 and Frame 1177 has StartTime = -417.509
// A positive integer representing number of minutes since the start of the run
// Theoretically, this will be the case for IMS_TOF_4 acquired after 12/14/2011
// ReSharper restore CommentTypo
double runTimeMinutes = 0;
// First examine globalParams.DateStarted
try
{
var validStartTime = false;
var reportedDateStartedText = globalParams.GetValueString(GlobalParamKeyType.DateStarted);
if (!DateTime.TryParse(reportedDateStartedText, out var reportedDateStarted))
{
// Invalid date; log a message
OnWarningEvent(".UIMF file has an invalid DateStarted value in table Global_Parameters: " + reportedDateStartedText + "; " +
"will use the time the datafile was last modified");
inaccurateStartTime = true;
}
else
{
if (reportedDateStarted.Year < 450)
{
// Some .UIMF files have DateStarted values represented by huge integers, e.g. 127805472000000000 or 129145004045937500; example: BATs_TS_01_c4_Eagle_10-02-06_0000
// These numbers are the number of ticks since 1 January 1601 (where each tick is 100 ns)
// This value is returned by function GetSystemTimeAsFileTime (see http://en.wikipedia.org/wiki/System_time)
// When SQLite parses these numbers, it converts them to years around 0410
// To get the correct year, simply add 1600
reportedDateStarted = reportedDateStarted.AddYears(1600);
validStartTime = true;
}
else if (reportedDateStarted.Year < 2000 || reportedDateStarted.Year > DateTime.Now.Year + 1)
{
// Invalid date; log a message
OnWarningEvent(".UIMF file has an invalid DateStarted value in table Global_Parameters: " + reportedDateStartedText + "; " +
"will use the time the datafile was last modified");
inaccurateStartTime = true;
}
else
{
validStartTime = true;
}
}
if (validStartTime)
{
datasetFileInfo.AcqTimeStart = reportedDateStarted;
// Update the end time to match the start time; we'll update it below using the start/end times obtained from the frame parameters
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
}
catch (Exception ex2)
{
OnWarningEvent("Exception extracting the DateStarted date from table Global_Parameters in the .UIMF file: " + ex2.Message);
}
// NumFrames is the total number of frames in the file (for all frame types)
datasetFileInfo.ScanCount = globalParams.NumFrames;
if (masterFrameNumList.Length > datasetFileInfo.ScanCount)
{
datasetFileInfo.ScanCount = masterFrameNumList.Length;
}
if (masterFrameNumList.Length > 0)
{
// Ideally, we would lookup the acquisition time of the first frame and the last frame, then subtract the two times to determine the run time
// However, given the odd values that can be present in the StartTime field, we need to construct a full list of start times and then parse it
// Get the start time of the first frame
// See above for the various numbers that could be stored in the StartTime column
var frameParams = uimfReader.GetFrameParams(masterFrameNumList[0]);
var startTime = frameParams.GetValueDouble(FrameParamKeyType.StartTimeMinutes);
// Get the start time of the last frame
// If the reported start time is zero, step back until a non-zero start time is reported
var frameIndex = masterFrameNumList.Length - 1;
double endTime;
do
{
frameParams = uimfReader.GetFrameParams(masterFrameNumList[frameIndex]);
endTime = frameParams.GetValueDouble(FrameParamKeyType.StartTimeMinutes);
if (Math.Abs(endTime) < float.Epsilon)
{
frameIndex -= 1;
}
} while (Math.Abs(endTime) < float.Epsilon && frameIndex >= 0);
// Check whether the StartTime and EndTime values are based on ticks
if (startTime >= 1E+17 && endTime > 1E+17)
{
// StartTime and EndTime were stored as the number of ticks (where each tick is 100 ns)
// Tick start date is either 1 January 1601 or 1 January 0001
var runTime = DateTime.MinValue.AddTicks((long)(endTime - startTime));
runTimeMinutes = runTime.Subtract(DateTime.MinValue).TotalMinutes;
// In some .UIMF files, the DateStarted column in Global_Parameters is simply the date, and not a specific time of day
// If that's the case, update datasetFileInfo.AcqTimeStart to be based on runTimeMinutes
if (datasetFileInfo.AcqTimeStart.Date == datasetFileInfo.AcqTimeStart)
{
var reportedDateStarted = DateTime.MinValue.AddTicks((long)startTime);
if (reportedDateStarted.Year < 500)
{
reportedDateStarted = reportedDateStarted.AddYears(1600);
}
if (reportedDateStarted.Year >= 2000 && reportedDateStarted.Year <= DateTime.Now.Year + 1)
{
// Date looks valid
if (inaccurateStartTime)
{
datasetFileInfo.AcqTimeStart = reportedDateStarted;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
else
{
// How does it compare to datasetFileInfo.AcqTimeStart?
if (reportedDateStarted.Subtract(datasetFileInfo.AcqTimeStart).TotalHours < 24)
{
// Update the date
datasetFileInfo.AcqTimeStart = reportedDateStarted;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
}
}
}
}
else
{
// Ideally, we'd just compute RunTime like this: runTimeMinutes = endTime - startTime
// But, given the idiosyncrasies that can occur, we need to construct a full list of start times
var startTimes = new List <double>();
double endTimeAddon = 0;
for (var index = 0; index <= masterFrameNumList.Length - 1; index++)
{
frameParams = uimfReader.GetFrameParams(masterFrameNumList[index]);
startTimes.Add(frameParams.GetValueDouble(FrameParamKeyType.StartTimeMinutes));
}
// ReSharper disable once CommentTypo
// Some datasets erroneously have zeroes stored in the .UIMF file for the StartTime of the last two frames; example: Sarc_MS2_26_2Apr11_Cheetah_11-02-18_inverse
// Check for this and remove them
var frameCountRemoved = 0;
while (Math.Abs(startTimes[startTimes.Count - 1]) < float.Epsilon)
{
startTimes.RemoveAt(startTimes.Count - 1);
frameCountRemoved += 1;
if (startTimes.Count == 0)
{
break;
}
}
if (frameCountRemoved > 0)
{
if (startTimes.Count > 2)
{
// Compute the amount of time (in minutes) to addon to the total run time
// We're computing the time between two frames, and multiplying that by frameCountRemoved
endTimeAddon += frameCountRemoved * (startTimes[startTimes.Count - 1] - startTimes[startTimes.Count - 2]);
}
}
// Now check for the StartTime changing to a smaller number from one frame to the next
// This could happen if the StartTime changed from 1439 to 0 as the system clock hits midnight
// Or if the StartTime changes from 59.9 to 0 as the system clock hits the top of a new hour
for (var index = 1; index <= startTimes.Count - 1; index++)
{
if (startTimes[index] < startTimes[index - 1])
{
if (startTimes[index - 1] > 1439)
{
endTimeAddon += 1440;
}
else if (startTimes[index - 1] > 59.7)
{
endTimeAddon += 60;
}
}
}
if (startTimes.Count > 0)
{
// Compute the runtime
// Luckily, even if startTime is -479.993 and entTime is -417.509, this works out to a positive, accurate runtime
endTime = startTimes[startTimes.Count - 1];
runTimeMinutes = endTime + endTimeAddon - startTime;
}
}
}
else
{
runTimeMinutes = 0;
}
if (runTimeMinutes > 0)
{
if (runTimeMinutes > 24000)
{
OnWarningEvent("Invalid runtime computed using the StartTime value from the first and last frames: " + runTimeMinutes);
}
else
{
if (inaccurateStartTime)
{
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd.AddMinutes(-runTimeMinutes);
}
else
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(runTimeMinutes);
}
}
}
}
catch (Exception ex)
{
OnWarningEvent("Exception extracting acquisition time information: " + ex.Message);
}
if (mSaveTICAndBPI || mCreateDatasetInfoFile || mCreateScanStatsFile || mSaveLCMS2DPlots)
{
// Load data from each frame
// This is used to create the TIC and BPI plot, the 2D LC/MS plot, and/or to create the Dataset Info File
LoadFrameDetails(uimfReader, dctMasterFrameList, masterFrameNumList);
}
if (mComputeOverallQualityScores)
{
// Note that this call will also create the TICs and BPIs
ComputeQualityScores(uimfReader, datasetFileInfo, dctMasterFrameList, masterFrameNumList);
}
}
// Close the handle to the data file
uimfReader?.Dispose();
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
// This will also compute the SHA-1 hash of the .UIMF file and add it to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetFileStats(uimfFile, datasetID);
// Copy over the updated file time info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetStatsSummarizerUsingDatasetFileInfo(datasetFileInfo);
PostProcessTasks();
return(!readError);
}
private bool ProcessChemstationMSDataFile(string datafilePath, DatasetFileInfo datasetFileInfo)
{
bool success;
var currentIndex = 0;
try
{
using (var reader = new ChemstationMSFileReader.clsChemstationDataMSFileReader(datafilePath))
{
datasetFileInfo.AcqTimeStart = reader.Header.AcqDate;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(reader.Header.RetentionTimeMinutesEnd);
datasetFileInfo.ScanCount = reader.Header.SpectraCount;
for (var spectrumIndex = 0; spectrumIndex <= datasetFileInfo.ScanCount - 1; spectrumIndex++)
{
currentIndex = spectrumIndex;
ChemstationMSFileReader.clsSpectralRecord spectrum = null;
List <float> mzList = null;
List <int> intensityList = null;
const int msLevel = 1;
bool validSpectrum;
try
{
reader.GetSpectrum(spectrumIndex, ref spectrum);
mzList = spectrum.Mzs;
intensityList = spectrum.Intensities;
validSpectrum = true;
}
catch (Exception ex)
{
OnWarningEvent("Exception obtaining data from the MS file for spectrum index " + currentIndex + ": " + ex.Message);
validSpectrum = false;
}
if (validSpectrum)
{
var scanStatsEntry = new ScanStatsEntry
{
ScanNumber = spectrumIndex + 1,
ScanType = msLevel,
ScanTypeName = "GC-MS",
ScanFilterText = string.Empty,
ElutionTime = spectrum.RetentionTimeMinutes.ToString("0.0###"),
TotalIonIntensity = StringUtilities.ValueToString(spectrum.TIC, 1),
BasePeakIntensity = StringUtilities.ValueToString(spectrum.BasePeakAbundance, 1),
BasePeakMZ = spectrum.BasePeakMZ.ToString("0.0###"),
BasePeakSignalToNoiseRatio = "0",
IonCount = mzList.Count
};
scanStatsEntry.IonCountRaw = scanStatsEntry.IonCount;
mDatasetStatsSummarizer.AddDatasetScan(scanStatsEntry);
if (mSaveTICAndBPI)
{
mTICAndBPIPlot.AddData(scanStatsEntry.ScanNumber, msLevel, spectrum.RetentionTimeMinutes, spectrum.BasePeakAbundance, spectrum.TIC);
if (mzList.Count > 0)
{
var ionsMZ = new double[mzList.Count];
var ionsIntensity = new double[mzList.Count];
for (var index = 0; index <= mzList.Count - 1; index++)
{
ionsMZ[index] = mzList[index];
ionsIntensity[index] = intensityList[index];
}
mLCMS2DPlot.AddScan(scanStatsEntry.ScanNumber, msLevel, spectrum.RetentionTimeMinutes, ionsMZ.Length, ionsMZ, ionsIntensity);
}
}
if (mCheckCentroidingStatus)
{
var mzDoubles = new List <double>(mzList.Count);
mzDoubles.AddRange(mzList.Select(ion => (double)ion));
mDatasetStatsSummarizer.ClassifySpectrum(mzDoubles, msLevel, "Scan " + scanStatsEntry.ScanNumber);
}
}
}
}
success = true;
}
catch (Exception ex)
{
// Exception reading file
OnWarningEvent("Exception reading data from the MS file at spectrum index " + currentIndex + ": " + ex.Message);
success = false;
}
return(success);
}
/// <summary>
/// Process the dataset
/// </summary>
/// <param name="dataFilePath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error or if the file has no scans</returns>
/// <remarks></remarks>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
ResetResults();
var dataFilePathLocal = string.Empty;
// Obtain the full path to the file
var rawFile = new FileInfo(dataFilePath);
if (!rawFile.Exists)
{
OnErrorEvent(".Raw file not found: " + dataFilePath);
return(false);
}
// Future, optional: Determine the DatasetID
// Unfortunately, this is not present in metadata.txt
// datasetID = LookupDatasetID(datasetName)
var datasetID = DatasetID;
// Record the file size and Dataset ID
datasetFileInfo.FileSystemCreationTime = rawFile.CreationTime;
datasetFileInfo.FileSystemModificationTime = rawFile.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetID = datasetID;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(rawFile.Name);
datasetFileInfo.FileExtension = rawFile.Extension;
datasetFileInfo.FileSizeBytes = rawFile.Length;
datasetFileInfo.ScanCount = 0;
mDatasetStatsSummarizer.ClearCachedData();
var deleteLocalFile = false;
var readError = false;
// Note: as of June 2018 this only works if you disable "Prefer 32-bit" when compiling as AnyCPU
// Use XRaw to read the .Raw file
// If reading from a SAMBA-mounted network share, and if the current user has
// Read privileges but not Read&Execute privileges, we will need to copy the file locally
var readerOptions = new ThermoReaderOptions
{
LoadMSMethodInfo = true,
LoadMSTuneInfo = true
};
var xcaliburAccessor = new XRawFileIO(readerOptions);
RegisterEvents(xcaliburAccessor);
// Open a handle to the data file
if (!xcaliburAccessor.OpenRawFile(rawFile.FullName))
{
// File open failed
OnErrorEvent("Call to .OpenRawFile failed for: " + rawFile.FullName);
ErrorCode = iMSFileInfoScanner.eMSFileScannerErrorCodes.ThermoRawFileReaderError;
readError = true;
if (!string.Equals(clsMSFileInfoScanner.GetAppDirectoryPath().Substring(0, 2), rawFile.FullName.Substring(0, 2), StringComparison.InvariantCultureIgnoreCase))
{
if (mCopyFileLocalOnReadError)
{
// Copy the file locally and try again
try
{
dataFilePathLocal = Path.Combine(clsMSFileInfoScanner.GetAppDirectoryPath(), Path.GetFileName(dataFilePath));
if (!string.Equals(dataFilePathLocal, dataFilePath, StringComparison.InvariantCultureIgnoreCase))
{
OnDebugEvent("Copying file " + Path.GetFileName(dataFilePath) + " to the working directory");
File.Copy(dataFilePath, dataFilePathLocal, true);
dataFilePath = string.Copy(dataFilePathLocal);
deleteLocalFile = true;
// Update rawFile then try to re-open
rawFile = new FileInfo(dataFilePath);
if (!xcaliburAccessor.OpenRawFile(rawFile.FullName))
{
// File open failed
OnErrorEvent("Call to .OpenRawFile failed for: " + rawFile.FullName);
ErrorCode = iMSFileInfoScanner.eMSFileScannerErrorCodes.ThermoRawFileReaderError;
readError = true;
}
else
{
readError = false;
}
}
}
catch (Exception)
{
readError = true;
}
}
}
}
if (!readError)
{
// Read the file info
try
{
datasetFileInfo.AcqTimeStart = xcaliburAccessor.FileInfo.CreationDate;
}
catch (Exception)
{
// Read error
readError = true;
}
if (!readError)
{
try
{
datasetFileInfo.ScanCount = xcaliburAccessor.GetNumScans();
if (datasetFileInfo.ScanCount > 0)
{
// Look up the end scan time then compute .AcqTimeEnd
var scanEnd = xcaliburAccessor.FileInfo.ScanEnd;
xcaliburAccessor.GetScanInfo(scanEnd, out clsScanInfo scanInfo);
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart.AddMinutes(scanInfo.RetentionTime);
}
else
{
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
}
}
catch (Exception)
{
// Error; use default values
datasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeStart;
datasetFileInfo.ScanCount = 0;
}
if (mSaveTICAndBPI || mCreateDatasetInfoFile || mCreateScanStatsFile ||
mSaveLCMS2DPlots || mCheckCentroidingStatus || MS2MzMin > 0)
{
// Load data from each scan
// This is used to create the TIC and BPI plot, the 2D LC/MS plot, and/or to create the Dataset Info File
LoadScanDetails(xcaliburAccessor);
}
if (mComputeOverallQualityScores)
{
// Note that this call will also create the TICs and BPIs
ComputeQualityScores(xcaliburAccessor, datasetFileInfo);
}
if (MS2MzMin > 0 && datasetFileInfo.ScanCount > 0)
{
// Verify that all of the MS2 spectra have m/z values below the required minimum
// Useful for validating that reporter ions can be detected
ValidateMS2MzMin();
}
}
}
mDatasetStatsSummarizer.SampleInfo.SampleName = xcaliburAccessor.FileInfo.SampleName;
mDatasetStatsSummarizer.SampleInfo.Comment1 = xcaliburAccessor.FileInfo.Comment1;
mDatasetStatsSummarizer.SampleInfo.Comment2 = xcaliburAccessor.FileInfo.Comment2;
// Add the devices
var deviceFilterList = new SortedSet <Device> {
Device.MS,
Device.MSAnalog
};
// First add the MS devices
AddThermoDevices(xcaliburAccessor, datasetFileInfo, deviceFilterList, new SortedSet <Device>());
// Now add any non-mass spec devices
AddThermoDevices(xcaliburAccessor, datasetFileInfo, new SortedSet <Device>(), deviceFilterList);
if (mSaveTICAndBPI && datasetFileInfo.DeviceList.Count > 0)
{
mInstrumentSpecificPlots.Clear();
foreach (var device in datasetFileInfo.DeviceList)
{
if (device.DeviceType == Device.MS || device.DeviceType == Device.MSAnalog)
{
continue;
}
var chromatogramData = xcaliburAccessor.GetChromatogramData(device.DeviceType, device.DeviceNumber);
if (chromatogramData.Count == 0)
{
continue;
}
var devicePlot = AddInstrumentSpecificPlot(device.DeviceDescription);
devicePlot.TICXAxisLabel = string.IsNullOrWhiteSpace(device.AxisLabelX) ? "Scan number" : device.AxisLabelX;
devicePlot.TICYAxisLabel = device.YAxisLabelWithUnits;
devicePlot.TICYAxisExponentialNotation = false;
devicePlot.DeviceType = device.DeviceType;
devicePlot.TICPlotAbbrev = string.Format("{0}{1}", device.DeviceType.ToString(), device.DeviceNumber);
devicePlot.TICAutoMinMaxY = true;
devicePlot.RemoveZeroesFromEnds = false;
float acqLengthMinutes;
if (datasetFileInfo.AcqTimeEnd > datasetFileInfo.AcqTimeStart)
{
acqLengthMinutes = (float)datasetFileInfo.AcqTimeEnd.Subtract(datasetFileInfo.AcqTimeStart).TotalMinutes;
}
else
{
acqLengthMinutes = chromatogramData.Count;
}
var dataCount = chromatogramData.Count;
foreach (var dataPoint in chromatogramData)
{
var scanNumber = dataPoint.Key;
float scanTimeMinutes;
if (acqLengthMinutes > 0)
{
scanTimeMinutes = scanNumber / (float)dataCount * acqLengthMinutes;
}
else
{
scanTimeMinutes = scanNumber;
}
devicePlot.AddDataTICOnly(scanNumber, 1, scanTimeMinutes, dataPoint.Value);
}
}
}
if (!string.IsNullOrEmpty(xcaliburAccessor.FileInfo.SampleComment))
{
if (string.IsNullOrEmpty(mDatasetStatsSummarizer.SampleInfo.Comment1))
{
mDatasetStatsSummarizer.SampleInfo.Comment1 = xcaliburAccessor.FileInfo.SampleComment;
}
else
{
if (string.IsNullOrEmpty(mDatasetStatsSummarizer.SampleInfo.Comment2))
{
mDatasetStatsSummarizer.SampleInfo.Comment2 = xcaliburAccessor.FileInfo.SampleComment;
}
else
{
// Append the sample comment to comment 2
mDatasetStatsSummarizer.SampleInfo.Comment2 += "; " + xcaliburAccessor.FileInfo.SampleComment;
}
}
}
// Close the handle to the data file
xcaliburAccessor.CloseRawFile();
// Read the file info from the file system
// (much of this is already in datasetFileInfo, but we'll call UpdateDatasetFileStats() anyway to make sure all of the necessary steps are taken)
// This will also compute the SHA-1 hash of the .Raw file and add it to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetFileStats(rawFile, datasetID);
// Copy over the updated file time info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetStatsSummarizerUsingDatasetFileInfo(datasetFileInfo);
// Delete the local copy of the data file
if (deleteLocalFile)
{
try
{
File.Delete(dataFilePathLocal);
}
catch (Exception)
{
// Deletion failed
OnErrorEvent("Deletion failed for: " + Path.GetFileName(dataFilePathLocal));
}
}
PostProcessTasks();
return(!readError);
}
private bool ParseAcqMethodFile(string directoryPath, DatasetFileInfo datasetFileInfo)
{
double totalRuntime = 0;
var runTimeFound = false;
bool success;
try
{
// Open the acqmeth.txt file
var filePath = Path.Combine(directoryPath, AGILENT_ACQ_METHOD_FILE);
if (!File.Exists(filePath))
{
return(false);
}
// Populate a dictionary var with the text strings for finding lines with runtime information
// Note that "Post Run" occurs twice in the file, so we use clsLineMatchSearchInfo.Matched to track whether or not the text has been matched
var dctRunTimeText = new Dictionary <string, clsLineMatchSearchInfo>
{
{ ACQ_METHOD_FILE_EQUILIBRATION_TIME_LINE, new clsLineMatchSearchInfo(true) },
{ ACQ_METHOD_FILE_RUN_TIME_LINE, new clsLineMatchSearchInfo(true) }
};
// We could also add in the "Post Run" time for determining total acquisition time, but we don't do this, to stay consistent with run times reported by the MS file
// dctRunTimeText.Add(ACQ_METHOD_FILE_POST_RUN_LINE, New clsLineMatchSearchInfo(False))
using (var reader = new StreamReader(filePath))
{
while (!reader.EndOfStream)
{
var dataLine = reader.ReadLine();
if (string.IsNullOrWhiteSpace(dataLine))
{
continue;
}
foreach (var key in dctRunTimeText.Keys)
{
if (dctRunTimeText[key].Matched)
{
continue;
}
bool matchSuccess;
if (dctRunTimeText[key].MatchLineStart)
{
matchSuccess = dataLine.StartsWith(key);
}
else
{
matchSuccess = dataLine.Contains(key);
}
if (!matchSuccess)
{
continue;
}
if (!ExtractRunTime(dataLine, out var runTime))
{
continue;
}
dctRunTimeText[key].Matched = true;
totalRuntime += runTime;
runTimeFound = true;
break;
}
}
}
success = runTimeFound;
}
catch (Exception ex)
{
// Exception reading file
OnErrorEvent("Exception reading " + AGILENT_ACQ_METHOD_FILE + ": " + ex.Message, ex);
success = false;
}
if (success)
{
// Update the acquisition start time
datasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeEnd.AddMinutes(-totalRuntime);
}
return(success);
}
/// <summary>
/// Process the dataset
/// </summary>
/// <param name="dataFilePath"></param>
/// <param name="datasetFileInfo"></param>
/// <returns>True if success, False if an error</returns>
/// <remarks></remarks>
public override bool ProcessDataFile(string dataFilePath, DatasetFileInfo datasetFileInfo)
{
var success = false;
var acqTimeDetermined = false;
ResetResults();
try
{
var agilentDFolder = new DirectoryInfo(dataFilePath);
var msDataFilePath = Path.Combine(agilentDFolder.FullName, AGILENT_MS_DATA_FILE);
datasetFileInfo.FileSystemCreationTime = agilentDFolder.CreationTime;
datasetFileInfo.FileSystemModificationTime = agilentDFolder.LastWriteTime;
// The acquisition times will get updated below to more accurate values
datasetFileInfo.AcqTimeStart = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.AcqTimeEnd = datasetFileInfo.FileSystemModificationTime;
datasetFileInfo.DatasetName = GetDatasetNameViaPath(agilentDFolder.Name);
datasetFileInfo.FileExtension = agilentDFolder.Extension;
datasetFileInfo.FileSizeBytes = 0;
// Look for the MS file
// Use its modification time to get an initial estimate for the acquisition end time
// Assign the .MS file's size to .FileSizeBytes
var msDatafile = new FileInfo(msDataFilePath);
if (msDatafile.Exists)
{
datasetFileInfo.FileSizeBytes = msDatafile.Length;
datasetFileInfo.AcqTimeStart = msDatafile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = msDatafile.LastWriteTime;
// Read the file info from the file system
// This will also compute the SHA-1 hash of the Data.MS file and add it to mDatasetStatsSummarizer.DatasetFileInfo
UpdateDatasetFileStats(msDatafile, datasetFileInfo.DatasetID);
success = true;
}
else
{
// Compute the hash of the largest file in the .D directory
AddLargestInstrumentFile(agilentDFolder);
}
datasetFileInfo.ScanCount = 0;
if (success)
{
// Read the detailed data from the MS file
success = ProcessChemstationMSDataFile(msDataFilePath, datasetFileInfo);
if (success)
{
acqTimeDetermined = true;
}
}
if (!success)
{
// MS file not found (or problems parsing); use acqmeth.txt and/or GC.ini
// The timestamp of the acqmeth.txt file or GC.ini file is more accurate than the GC.ini file, so we'll use that
var methodFile = new FileInfo(Path.Combine(agilentDFolder.FullName, AGILENT_ACQ_METHOD_FILE));
if (!methodFile.Exists)
{
methodFile = new FileInfo(Path.Combine(agilentDFolder.FullName, AGILENT_GC_INI_FILE));
}
if (methodFile.Exists)
{
// ReSharper disable once CommentTypo
// Update the AcqTimes only if the LastWriteTime of the acqmeth.txt or GC.ini file is within the next 60 minutes of .AcqTimeEnd
if (!success || methodFile.LastWriteTime.Subtract(datasetFileInfo.AcqTimeEnd).TotalMinutes < 60)
{
datasetFileInfo.AcqTimeStart = methodFile.LastWriteTime;
datasetFileInfo.AcqTimeEnd = methodFile.LastWriteTime;
success = true;
}
if (datasetFileInfo.FileSizeBytes == 0)
{
// File size was not determined from the MS file
// Instead, sum up the sizes of all of the files in this directory
foreach (var item in agilentDFolder.GetFiles())
{
datasetFileInfo.FileSizeBytes += item.Length;
}
mDatasetStatsSummarizer.DatasetFileInfo.FileSizeBytes = datasetFileInfo.FileSizeBytes;
}
}
}
if (!acqTimeDetermined)
{
try
{
// Parse the acqmeth.txt file to determine the actual values for .AcqTimeStart and .AcqTimeEnd
success = ParseAcqMethodFile(dataFilePath, datasetFileInfo);
if (!success)
{
// Try to extract Runtime from the GC.ini file
success = ParseGCIniFile(dataFilePath, datasetFileInfo);
}
}
catch (Exception)
{
// Error parsing the acqmeth.txt file or GC.in file; do not abort
}
// We set success to true, even if either of the above functions fail
success = true;
}
if (success)
{
// Copy over the updated file time info and scan info from datasetFileInfo to mDatasetStatsSummarizer.DatasetFileInfo
mDatasetStatsSummarizer.DatasetFileInfo.DatasetName = string.Copy(datasetFileInfo.DatasetName);
mDatasetStatsSummarizer.DatasetFileInfo.FileExtension = string.Copy(datasetFileInfo.FileExtension);
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeStart = datasetFileInfo.AcqTimeStart;
mDatasetStatsSummarizer.DatasetFileInfo.AcqTimeEnd = datasetFileInfo.AcqTimeEnd;
mDatasetStatsSummarizer.DatasetFileInfo.ScanCount = datasetFileInfo.ScanCount;
}
}
catch (Exception ex)
{
OnErrorEvent("Exception parsing GC .D directory: " + ex.Message, ex);
success = false;
}
PostProcessTasks();
return(success);
}
public ModelObject createEmptyFile(String datasetType, String datasetName, String filePath, String fileRefName, ModelObject container, String relationType)
{
var dmService = DataManagementService.getService(Session.getConnection());
FileManagementUtility fmsFileManagement = new FileManagementUtility(Session.getConnection(), null, null, new[] { cfg.get("Fms_BootStrap_Urls") }, cfg.get("file_temp"));
//FileManagementUtility fmsFileManagement = new FileManagementUtility(Session.getConnection());
// Create a Dataset
DatasetProperties2 props = new DatasetProperties2();
//props.ClientId = "datasetWriteTixTestClientId";
//props.Type = "Text";
//props.Name = "Sample-FMS-Upload-maxtt";
//props.Description = "Testing put File";
props.ClientId = datasetName + "ClientId";
props.Type = datasetType;
props.Name = datasetName;
props.Container = container;
props.RelationType = relationType;
CreateDatasetsResponse resp = dmService.CreateDatasets2(new DatasetProperties2[] { props });
// Create a file to associate with dataset
var file = new FileInfo(filePath);
if (!file.Exists)
{
throw new Exception("要上传的文件不存在，上传文件失败！");
}
DatasetFileInfo fileInfo = new DatasetFileInfo();
fileInfo.ClientId = "file_1";
//fileInfo.FileName = "./template/url.txt";
fileInfo.FileName = file.FullName;
fileInfo.NamedReferencedName = fileRefName;
fileInfo.IsText = false;
fileInfo.AllowReplace = false;
DatasetFileInfo[] fileInfos = { fileInfo };
GetDatasetWriteTicketsInputData inputData = new GetDatasetWriteTicketsInputData();
inputData.Dataset = resp.Output[0].Dataset;
inputData.CreateNewVersion = false;
inputData.DatasetFileInfos = fileInfos;
GetDatasetWriteTicketsInputData[] inputs = { inputData };
ServiceData response = fmsFileManagement.PutFiles(inputs);
if (response.sizeOfPartialErrors() > 0)
{
System.Console.Out.WriteLine("FileManagementService single upload returned partial errrors: " + response.sizeOfPartialErrors());
}
//add relation
ModelObject datasets = inputs[0].Dataset;
return(datasets);
}
/// <summary>
/// This function is used to determine one or more overall quality scores
/// </summary>
/// <param name="uimfReader"></param>
/// <param name="datasetFileInfo"></param>
/// <param name="dctMasterFrameList"></param>
/// <param name="masterFrameNumList"></param>
private void ComputeQualityScores(
DataReader uimfReader,
DatasetFileInfo datasetFileInfo,
IReadOnlyDictionary <int, UIMFData.FrameType> dctMasterFrameList,
IReadOnlyList <int> masterFrameNumList)
{
float overallScore;
double overallAvgIntensitySum = 0;
var overallAvgCount = 0;
if (mLCMS2DPlot.ScanCountCached > 0)
{
// Obtain the overall average intensity value using the data cached in mLCMS2DPlot
// This avoids having to reload all of the data using uimfReader
const int msLevelFilter = 1;
overallScore = mLCMS2DPlot.ComputeAverageIntensityAllScans(msLevelFilter);
}
else
{
var globalParams = uimfReader.GetGlobalParams();
var globalMaxBins = globalParams.Bins;
var mzList = new double[globalMaxBins + 1];
var intensityList = new int[globalMaxBins + 1];
// Call .GetStartAndEndScans to get the start and end Frames
GetStartAndEndScans(globalParams.NumFrames, out var frameStart, out var frameEnd);
for (var masterFrameNumIndex = 0; masterFrameNumIndex <= masterFrameNumList.Count - 1; masterFrameNumIndex++)
{
var frameNumber = masterFrameNumList[masterFrameNumIndex];
if (!dctMasterFrameList.TryGetValue(frameNumber, out var eFrameType))
{
OnWarningEvent(string.Format(
"Frametype {0} not found in dictionary dctMasterFrameList; ignoring frame {1} in ComputeQualityScores",
eFrameType, frameNumber));
continue;
}
// Check whether the frame number is within the desired range
if (frameNumber < frameStart || frameNumber > frameEnd)
{
continue;
}
FrameParams frameParams;
try
{
frameParams = uimfReader.GetFrameParams(frameNumber);
}
catch (Exception)
{
Console.WriteLine("Exception obtaining frame parameters for frame " + frameNumber + "; will skip this frame");
frameParams = null;
}
if (frameParams == null)
{
continue;
}
// We have to clear the m/z and intensity arrays before calling GetSpectrum
Array.Clear(mzList, 0, mzList.Length);
Array.Clear(intensityList, 0, intensityList.Length);
// Process all of the IMS scans in this Frame to compute a summed spectrum representative of the frame
// Scans likely range from 0 to frameParams.Scans-1, but we'll use frameParams.Scans just to be safe
var ionCount = uimfReader.GetSpectrum(frameNumber, frameNumber, eFrameType, 0, frameParams.Scans, out mzList, out intensityList);
if (ionCount <= 0)
{
continue;
}
// The m/z and intensity arrays might contain entries with m/z values of 0;
// need to copy the data in place to get the data in the correct format.
if (ionCount > mzList.Length)
{
ionCount = mzList.Length;
}
var targetIndex = 0;
for (var ionIndex = 0; ionIndex <= ionCount - 1; ionIndex++)
{
if (mzList[ionIndex] > 0)
{
if (targetIndex != ionIndex)
{
mzList[targetIndex] = mzList[ionIndex];
intensityList[targetIndex] = intensityList[ionIndex];
}
targetIndex += 1;
}
}
ionCount = targetIndex;
if (ionCount > 0)
{
// ToDo: Analyze ionMZ and ionIntensity to compute a quality scores
// Keep track of the quality scores and then store one or more overall quality scores in datasetFileInfo.OverallQualityScore
// For now, this just computes the average intensity for each scan and then computes and overall average intensity value
double intensitySum = 0;
for (var ionIndex = 0; ionIndex <= ionCount - 1; ionIndex++)
{
intensitySum += intensityList[ionIndex];
}
overallAvgIntensitySum += intensitySum / ionCount;
overallAvgCount += 1;
}
}
if (overallAvgCount > 0)
{
overallScore = (float)(overallAvgIntensitySum / overallAvgCount);
}
else
{
overallScore = 0;
}
}
datasetFileInfo.OverallQualityScore = overallScore;
}
