private void GenerateChromatograms(ChromatogramTask chromatogramTask)
{
int totalAnalyses = chromatogramTask.AnalysisChromatograms.Count;
if (totalAnalyses == 0)
{
return;
}
if (!UpdateProgress(chromatogramTask, 0))
{
return;
}
var msDataFile = chromatogramTask.MsDataFile;
MsDataFileUtil.InitMsDataFile(chromatogramTask.Workspace, msDataFile);
var analyses = new List<AnalysisChromatograms>(chromatogramTask.AnalysisChromatograms);
MsDataFileImpl pwizMsDataFileImpl;
var path = _workspace.GetDataFilePath(msDataFile.Name);
try
{
pwizMsDataFileImpl = new MsDataFileImpl(path);
}
catch (Exception exception)
{
ErrorHandler.LogException("Chromatogram Generator", "Error opening " + path, exception);
_workspace.RejectMsDataFile(msDataFile);
return;
}
using (pwizMsDataFileImpl)
{
var completeAnalyses = new List<AnalysisChromatograms>();
int totalScanCount = pwizMsDataFileImpl.SpectrumCount;
var lastTimeInDataFile =
chromatogramTask.MsDataFile.GetTime(chromatogramTask.MsDataFile.GetSpectrumCount() - 1);
double minTime = lastTimeInDataFile;
double maxTime = msDataFile.GetTime(0);
foreach (var analysis in analyses)
{
minTime = Math.Min(minTime, analysis.FirstTime);
maxTime = Math.Max(maxTime, analysis.LastTime);
}
int firstScan = msDataFile.FindScanIndex(minTime);
for (int iScan = firstScan; analyses.Count > 0 && iScan < totalScanCount; iScan++)
{
double time = msDataFile.GetTime(iScan);
int progress = (int)(100 * (time - minTime) / (maxTime - minTime));
progress = Math.Min(progress, 100);
progress = Math.Max(progress, 0);
if (!UpdateProgress(chromatogramTask, progress))
{
return;
}
List<AnalysisChromatograms> activeAnalyses = new List<AnalysisChromatograms>();
double nextTime = Double.MaxValue;
if (msDataFile.GetMsLevel(iScan, pwizMsDataFileImpl) != 1)
{
continue;
}
foreach (var analysis in analyses)
{
nextTime = Math.Min(nextTime, analysis.FirstTime);
if (analysis.FirstTime <= time)
{
activeAnalyses.Add(analysis);
}
}
if (activeAnalyses.Count == 0)
{
int nextScan = msDataFile.FindScanIndex(nextTime);
iScan = Math.Max(iScan, nextScan - 1);
continue;
}
bool lowMemory = IsLowOnMemory();
// If we have exceeded the number of analyses we should be working on at once,
// throw out any that we haven't started.
for (int iAnalysis = activeAnalyses.Count - 1; iAnalysis >= 0
&& (activeAnalyses.Count > _maxConcurrentAnalyses || lowMemory); iAnalysis--)
{
var analysis = activeAnalyses[iAnalysis];
if (analysis.Times.Count > 0)
{
continue;
}
activeAnalyses.RemoveAt(iAnalysis);
analyses.Remove(analysis);
}
double[] mzArray, intensityArray;
pwizMsDataFileImpl.GetSpectrum(iScan, out mzArray, out intensityArray);
if (!pwizMsDataFileImpl.IsCentroided(iScan))
{
var centroider = new Centroider(mzArray, intensityArray);
centroider.GetCentroidedData(out mzArray, out intensityArray);
}
foreach (var analysis in activeAnalyses)
{
var points = new List<ChromatogramPoint>();
foreach (var chromatogram in analysis.Chromatograms)
{
points.Add(MsDataFileUtil.GetPoint(chromatogram.MzRange, mzArray, intensityArray));
}
analysis.AddPoints(iScan, time, points);
}
var incompleteAnalyses = new List<AnalysisChromatograms>();
foreach (var analysis in analyses)
{
if (analysis.LastTime <= time)
{
completeAnalyses.Add(analysis);
}
else
{
incompleteAnalyses.Add(analysis);
}
}
SaveChromatograms(chromatogramTask, completeAnalyses, false);
completeAnalyses.Clear();
analyses = incompleteAnalyses;
}
completeAnalyses.AddRange(analyses);
SaveChromatograms(chromatogramTask, completeAnalyses, true);
}
}
public SpectrumData(MsDataFileImpl msDataFileImpl, int scanIndex)
{
ScanIndex = scanIndex;
var spectrum = msDataFileImpl.GetSpectrum(scanIndex);
if (spectrum.Centroided)
{
CentroidMzs = spectrum.Mzs;
CentroidIntensities = spectrum.Intensities;
}
else
{
ProfileMzs = spectrum.Mzs;
ProfileIntensities = spectrum.Intensities;
var centroider = new Centroider(ProfileMzs, ProfileIntensities);
double[] centroidMzs, centroidIntensities;
centroider.GetCentroidedData(out centroidMzs, out centroidIntensities);
CentroidMzs = centroidMzs;
CentroidIntensities = centroidIntensities;
}
Time = spectrum.RetentionTime;
MsLevel = spectrum.Level;
}
public void TestMsx(SrmDocument doc, string dataPath)
{
// Load the file and check MsDataFileImpl
using (var file = new MsDataFileImpl(dataPath))
{
var filter = new SpectrumFilter(doc, null, null);
Assert.IsTrue(filter.EnabledMsMs);
var demultiplexer = new MsxDemultiplexer(file, filter);
demultiplexer.ForceInitializeFile();
// Check that the demultiplexer found the correct multiplexing parameters
Assert.AreEqual(5, demultiplexer.IsoWindowsPerScan);
Assert.AreEqual(100, demultiplexer.NumIsoWindows);
Assert.AreEqual(20, demultiplexer.DutyCycleLength);
var isoMapper = demultiplexer.IsoMapperTest;
Assert.AreEqual(100, isoMapper.NumWindows);
// Check the isolation window mapper responsible for mapping each unique isolation
// window detected in the file to a unique index
TestIsolationWindowMapper(isoMapper, file, 4.00182);
var transBinner = new TransitionBinner(filter, isoMapper);
// Test creation of a transition binner from a spectrum filter
TestTransitionBinnerFromFilter(transBinner, isoMapper);
var testSpectrum = file.GetSpectrum(TEST_SPECTRUM);
// Generate a transition binner containing a lot of tough cases with
// overlapping transitions and test
double[] binnedExpected =
{
0.0, 25160.11261, 18254.06375, 18254.06375, 18254.06375,
11090.00577, 19780.18628, 19780.18628
};
var transBinnerOverlap = TestTransitionBinnerOverlap(testSpectrum, isoMapper, binnedExpected);
TestSpectrumProcessor(transBinnerOverlap, isoMapper, file, TEST_SPECTRUM);
TestPeakIntensityCorrection(testSpectrum, isoMapper,
demultiplexer);
int[] intensityIndices = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 290, 291, 292, 293, 294, 295, 296 };
double[] intensityValues =
{
0.0, 0.0, 0.0, 0.0, 142.95, 349.75,
542.87, 511.77, 248.4, 0.0, 49.28,
1033.65, 278.56, 0.0, 0.0, 0.0,
0.0, 0.0
};
// Test demultiplexing of a real spectrum
TestSpectrumDemultiplex(demultiplexer, TEST_SPECTRUM, testSpectrum, intensityIndices, intensityValues, 0);
}
}
private static void TestSpectrumProcessor(TransitionBinner transBinner,AbstractIsoWindowMapper isoMapper,
MsDataFileImpl file, int testSpectrumNum)
{
int lastSpectrum = testSpectrumNum + 60;
int firstSpectrum = testSpectrumNum - 60;
var spectrumProcessor = new SpectrumProcessor(lastSpectrum + 1, isoMapper, transBinner);
// Make a new spectrum processor using the testing constructor that takes in
// a TransitionBinner, which will be the one used for overlap
// Add the spectra to the cache +/- 60 from the spectrum of interest
for (int i = firstSpectrum; i <= lastSpectrum; ++i)
{
var spectrum = file.GetSpectrum(i);
spectrumProcessor.AddSpectrum(i, spectrum);
}
// Check that the caching worked correctly
for (int spectrumIndex = firstSpectrum; spectrumIndex <= lastSpectrum; ++spectrumIndex)
{
ScanCached specData;
Assert.IsTrue(spectrumProcessor.TryGetSpectrum(spectrumIndex, out specData));
// Check that the cached processing results match the output of
// redoing the processing manually by extracting the spectrum from
// the file again and using transBinner to bin the data
var testSpectrum = file.GetSpectrum(spectrumIndex);
double[] expectedBinnedData = new double[transBinner.NumBins];
transBinner.BinData(testSpectrum.Mzs, testSpectrum.Intensities, ref expectedBinnedData);
ScanCached testSpecData;
spectrumProcessor.TryGetSpectrum(spectrumIndex, out testSpecData);
var seenBinnedData = testSpecData.Data;
Assert.AreEqual(expectedBinnedData.Length, seenBinnedData.Length);
for (int i = 0; i < expectedBinnedData.Length; ++i)
{
Assert.AreEqual(expectedBinnedData[i], seenBinnedData[i], 0.0001);
}
}
}
private static void TestIsolationWindowMapper(AbstractIsoWindowMapper isoMapper, MsDataFileImpl file, double width)
{
// Check the first five spectra in the file
for (int checkIndex = 0 ; checkIndex < 5; ++checkIndex)
{
var spectrum = file.GetSpectrum(checkIndex);
var precursors = spectrum.Precursors;
List<int> isoIndices = new List<int>();
// Make sure that each precursor read from the file is included in the isolation
// window mapper
foreach (var precursor in precursors)
{
int index;
Assert.IsTrue(precursor.IsolationMz.HasValue);
Assert.IsTrue(isoMapper.TryGetWindowIndex(precursor.IsolationMz.Value, out index));
isoIndices.Add(index);
}
int[] isoIndicesFromMapper;
int[] overlapIndicesFromMapper;
double[] mask = new double[isoMapper.NumWindows];
// Make sure the isolation windows called from GetWindowMask match those
// extracted from the spectrum manually above using TryGetWindowIndex
isoMapper.GetWindowMask(spectrum, out isoIndicesFromMapper, out overlapIndicesFromMapper, ref mask);
Assert.AreEqual(isoIndices.Count,isoIndicesFromMapper.Length);
foreach (int index in isoIndicesFromMapper)
{
Assert.IsTrue(isoIndices.Contains(index));
}
// Make sure the overlapIndicesFromMapper matches the mask
for (int i = 0; i < mask.Length; ++i)
{
if (mask[i] < 0.5)
Assert.AreEqual(0.0, mask[i]);
else
{
Assert.AreEqual(1.0, mask[i]);
Assert.IsTrue(overlapIndicesFromMapper.Contains(i));
}
}
}
var testMzs = new[] {553.49, 623.7, 859, 658.55, 768.7, 621.52};
// For each test m/z, get the matching isolation window from isoMapper and
// make sure the m/z does indeed fall within the window
foreach (var checkMz in testMzs)
{
int windowIndex;
Assert.IsTrue(isoMapper.TryGetWindowFromMz(checkMz, out windowIndex));
var matchWindowCenterMz = isoMapper.GetPrecursor(windowIndex);
Assert.IsTrue(matchWindowCenterMz.IsolationMz.HasValue);
// Check that mz is within window (width/2 )
Assert.IsTrue(Math.Abs(matchWindowCenterMz.IsolationMz.Value - checkMz) < width/2);
}
// Test out of range precursors
int dummyIndex;
Assert.IsFalse(isoMapper.TryGetDeconvFromMz(315.25, out dummyIndex));
Assert.AreEqual(-1, dummyIndex);
Assert.IsFalse(isoMapper.TryGetDeconvFromMz(1005.2, out dummyIndex));
Assert.AreEqual(-1, dummyIndex);
}
public void TestOverlap(SrmDocument doc, string dataPath)
{
// Load the file and check MsDataFileImpl
using (var file = new MsDataFileImpl(dataPath))
{
var filter = new SpectrumFilter(doc, null, null);
Assert.IsTrue(filter.EnabledMsMs);
var demultiplexer = new OverlapDemultiplexer(file, filter);
demultiplexer.ForceInitializeFile();
// Check that the demultiplexer found the correct multiplexing parameters
Assert.AreEqual(1, demultiplexer.IsoWindowsPerScan);
Assert.AreEqual(40, demultiplexer.NumIsoWindows);
Assert.AreEqual(41, demultiplexer.NumDeconvRegions);
var isoMapper = demultiplexer.IsoMapperTest;
// Basic checks of IsolationWindowMapper
TestIsolationWindowMapper(isoMapper, file, 20.009);
// Checks of overlap-specific functionality in IsolationWindowMapper
TestOverlapIsolationWindowMapper(isoMapper, file);
var transBinner = new TransitionBinner(filter, isoMapper);
// Test creation of a transition binner from a spectrum filter
TestTransitionBinnerFromFilter(transBinner, isoMapper);
var testSpectrum = file.GetSpectrum(TEST_SPECTRUM_OVERLAP);
// Generate a transition binner containing a lot of tough cases with
// overlapping transitions and test
double[] binnedExpected =
{
0.0, 0.0, 0.0, 0.0, 0.0,
0.0, 0.0, 0.0
};
var transBinnerOverlap = TestTransitionBinnerOverlap(testSpectrum, isoMapper, binnedExpected);
TestSpectrumProcessor(transBinnerOverlap, isoMapper, file, TEST_SPECTRUM_OVERLAP);
int[] intensityIndices = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 290, 291, 292, 293, 294, 295, 296};
double[] intensityValues =
{
0.0, 0.0, 0.0, 0.0, 4545.85, 15660.49,
35050.01, 56321.66, 62715.75, 43598.31, 23179.42,
2745.94, 3870.54, 4060.16, 3148.17, 1656.38,
0.0, 0.0
};
// Test demultiplexing of a real spectrum
TestSpectrumDemultiplex(demultiplexer, TEST_SPECTRUM_OVERLAP, testSpectrum, intensityIndices,
intensityValues, 0);
}
}
protected void TestUsePredictedTime()
{
const double FILTER_LENGTH = 2.7;
RunUI(() => SkylineWindow.OpenFile(TestFilesDir.GetTestPath("RetentionTimeFilterTest.sky")));
WaitForDocumentLoaded();
RunUI(() => SkylineWindow.SaveDocument(TestFilesDir.GetTestPath("TestUsePredictedTime.sky")));
SetUiDocument(ChangeFullScanSettings(SkylineWindow.Document, SkylineWindow.Document.Settings.TransitionSettings.FullScan
.ChangeRetentionTimeFilter(RetentionTimeFilterType.scheduling_windows, FILTER_LENGTH)));
Assert.IsNull(SkylineWindow.Document.Settings.PeptideSettings.Prediction.RetentionTime);
Assert.IsFalse(SkylineWindow.Document.Settings.PeptideSettings.Prediction.UseMeasuredRTs);
// When we try to import a file, we should get an error about not having a peptide prediction algorithm
var messageDlg = ShowDialog<MessageDlg>(() => SkylineWindow.ImportResults());
RunUI(() =>
{
Assert.AreEqual(Resources.SkylineWindow_CheckRetentionTimeFilter_NoPredictionAlgorithm, messageDlg.Message);
messageDlg.Close();
});
var ssrCalcRegression = new RetentionTimeRegression("SSRCALC_FOR_RtFilterTest",
new RetentionScoreCalculator(RetentionTimeRegression.SSRCALC_100_A), .63,
5.8, 1.4, new MeasuredRetentionTime[0]);
// Now give the document a prediction algorithm
SetUiDocument(ChangePeptidePrediction(SkylineWindow.Document, SkylineWindow.Document.Settings.PeptideSettings.
Prediction.ChangeRetentionTime(ssrCalcRegression)));
// Now import two result files
{
var importResultsDlg = ShowDialog<ImportResultsDlg>(SkylineWindow.ImportResults);
var openDataSourceDialog = ShowDialog<OpenDataSourceDialog>(importResultsDlg.OkDialog);
RunUI(() =>
{
openDataSourceDialog.SelectFile("200fmol" + extension);
openDataSourceDialog.SelectFile("20fmol" + extension);
});
OkDialog(openDataSourceDialog, openDataSourceDialog.Open);
}
WaitForResultsImport();
{
var document = WaitForDocumentLoaded();
foreach (var chromatogramSet in document.Settings.MeasuredResults.Chromatograms)
{
foreach (var tuple in LoadAllChromatograms(document, chromatogramSet))
{
var peptide = tuple.Item1;
if (!peptide.IsProteomic)
continue;
var transitionGroup = tuple.Item2;
var predictedRetentionTime = ssrCalcRegression.GetRetentionTime(
document.Settings.GetModifiedSequence(peptide.Peptide.Sequence,
transitionGroup.TransitionGroup.LabelType, peptide.ExplicitMods)).Value;
AssertChromatogramWindow(document, chromatogramSet,
predictedRetentionTime - FILTER_LENGTH,
predictedRetentionTime + FILTER_LENGTH, tuple.Item3);
}
}
}
ChromatogramSet chromSetForScheduling = SkylineWindow.Document.Settings.MeasuredResults.Chromatograms[1];
// Create a SrmDocument with just the one ChromatogramSet that we are going to use for scheduling, so that
// we can assert later that the chromatogram windows are where this document says they should be.
SrmDocument documentForScheduling =
SkylineWindow.Document.ChangeMeasuredResults(
SkylineWindow.Document.Settings.MeasuredResults.ChangeChromatograms(new[] {chromSetForScheduling}));
SetUiDocument(ChangePeptidePrediction(SkylineWindow.Document, SkylineWindow.Document.Settings.PeptideSettings
.Prediction.ChangeUseMeasuredRTs(true).ChangeRetentionTime(null)));
{
var chooseSchedulingReplicatesDlg = ShowDialog<ChooseSchedulingReplicatesDlg>(SkylineWindow.ImportResults);
// Choose a scheduling replicate (the one saved above)
RunUI(() => Assert.IsTrue(chooseSchedulingReplicatesDlg.TrySetReplicateChecked(
chromSetForScheduling, true)));
var importResultsDlg = ShowDialog<ImportResultsDlg>(chooseSchedulingReplicatesDlg.OkDialog);
var openDataSourceDialog = ShowDialog<OpenDataSourceDialog>(importResultsDlg.OkDialog);
RunUI(()=>openDataSourceDialog.SelectFile("40fmol" + extension));
OkDialog(openDataSourceDialog, openDataSourceDialog.Open);
}
WaitForResultsImport();
{
var document = WaitForDocumentLoaded();
var chromatogramSet = document.Settings.MeasuredResults.Chromatograms.First(cs => cs.Name == "40fmol");
int countNull = 0;
foreach (var tuple in LoadAllChromatograms(document, chromatogramSet))
{
var prediction = new PeptidePrediction(null, null, true, 1, false, 0);
double windowRtIgnored;
var schedulingPeptide =
documentForScheduling.Molecules.First(pep => ReferenceEquals(pep.Peptide, tuple.Item1.Peptide));
var schedulingTransitionGroup = (TransitionGroupDocNode) schedulingPeptide.FindNode(tuple.Item2.TransitionGroup);
double? predictedRt = prediction.PredictRetentionTime(documentForScheduling,
schedulingPeptide,
schedulingTransitionGroup,
null, ExportSchedulingAlgorithm.Average, true, out windowRtIgnored);
if (!predictedRt.HasValue)
{
countNull++;
continue;
}
AssertChromatogramWindow(document, chromatogramSet, predictedRt.Value - FILTER_LENGTH, predictedRt.Value + FILTER_LENGTH, tuple.Item3);
}
Assert.AreEqual((TestSmallMolecules ? 1 : 0), countNull);
}
// Test using iRT with auto-calculated regression
{
const string calcName = "TestCalculator";
const string regressionName = "TestCalculatorAutoCalcRegression";
var peptideSettingsDlg = ShowDialog<PeptideSettingsUI>(SkylineWindow.ShowPeptideSettingsUI);
var editIrtDlg = ShowDialog<EditIrtCalcDlg>(peptideSettingsDlg.AddCalculator);
RunUI(() =>
{
editIrtDlg.OpenDatabase(TestFilesDir.GetTestPath("RetentionTimeFilterTest.irtdb"));
editIrtDlg.CalcName = calcName;
});
SkylineWindow.BeginInvoke(new Action(editIrtDlg.OkDialog));
var multiButtonMsgDlg = WaitForOpenForm<MultiButtonMsgDlg>();
OkDialog(multiButtonMsgDlg, ()=>multiButtonMsgDlg.DialogResult = DialogResult.Yes);
var editRtDlg = ShowDialog<EditRTDlg>(peptideSettingsDlg.AddRTRegression);
RunUI(() =>
{
editRtDlg.ChooseCalculator(calcName);
editRtDlg.SetAutoCalcRegression(true);
editRtDlg.SetRegressionName(regressionName);
editRtDlg.SetTimeWindow(1.0);
});
OkDialog(editRtDlg, editRtDlg.OkDialog);
RunUI(() =>
{
peptideSettingsDlg.ChooseRegression(regressionName);
peptideSettingsDlg.UseMeasuredRT(false);
});
OkDialog(peptideSettingsDlg, peptideSettingsDlg.OkDialog);
Assert.IsFalse(SkylineWindow.Document.Settings.PeptideSettings.Prediction.UseMeasuredRTs);
var importResultsDlg = ShowDialog<ImportResultsDlg>(SkylineWindow.ImportResults);
var openDataSourceDialog = ShowDialog<OpenDataSourceDialog>(importResultsDlg.OkDialog);
RunUI(() => openDataSourceDialog.SelectFile("8fmol" + extension));
OkDialog(openDataSourceDialog, openDataSourceDialog.Open);
WaitForResultsImport();
var document = WaitForDocumentLoaded();
var chromatogramSet = document.Settings.MeasuredResults.Chromatograms.First(cs => cs.Name == "8fmol");
var regressionLine =
document.Settings.PeptideSettings.Prediction.RetentionTime.GetConversion(
chromatogramSet.MSDataFileInfos.First().FileId);
var calculator = document.Settings.PeptideSettings.Prediction.RetentionTime.Calculator;
double fullGradientStartTime;
double fullGradientEndTime;
using (var msDataFile = new MsDataFileImpl(TestFilesDir.GetTestPath("8fmol" + extension)))
{
fullGradientStartTime = msDataFile.GetSpectrum(0).RetentionTime.Value;
fullGradientEndTime = msDataFile.GetSpectrum(msDataFile.SpectrumCount - 1).RetentionTime.Value;
}
foreach (var tuple in LoadAllChromatograms(document, chromatogramSet))
{
if (tuple.Item1.GlobalStandardType != PeptideDocNode.STANDARD_TYPE_IRT)
{
double? score =
calculator.ScoreSequence(document.Settings.GetModifiedSequence(tuple.Item1.Peptide.Sequence,
tuple.Item2.TransitionGroup.LabelType, tuple.Item1.ExplicitMods));
if (score.HasValue)
{
double? predictedRt = regressionLine.GetY(score.Value);
AssertChromatogramWindow(document, chromatogramSet, predictedRt.Value - FILTER_LENGTH,
predictedRt.Value + FILTER_LENGTH, tuple.Item3);
}
}
else
{
// IRT Standards get extracted for the full gradient
AssertChromatogramWindow(document, chromatogramSet, fullGradientStartTime, fullGradientEndTime, tuple.Item3);
}
}
}
}