//add possible protein/peptide terminal modifications that aren't on the terminal amino acids
//The purpose is for terminal mods that are contained WITHIN the Single peptide
private void AddInteriorTerminalModsToPrecursorIndex(List <int>[] precursorIndex, List <Product> fragmentMasses, PeptideWithSetModifications peptide, int peptideId, List <Modification> variableModifications)
{
//Get database annotated mods
Dictionary <int, List <Modification> > databaseAnnotatedMods = NonSpecificEnzymeSearchEngine.GetTerminalModPositions(peptide, CommonParameters.DigestionParams, variableModifications);
foreach (KeyValuePair <int, List <Modification> > relevantDatabaseMod in databaseAnnotatedMods)
{
int fragmentNumber = relevantDatabaseMod.Key;
Product fragmentAtIndex = fragmentMasses.Where(x => x.TerminusFragment.FragmentNumber == fragmentNumber).FirstOrDefault();
double basePrecursorMass = fragmentAtIndex == null ? peptide.MonoisotopicMass : fragmentAtIndex.NeutralMass - DissociationTypeCollection.GetMassShiftFromProductType(fragmentAtIndex.ProductType) + WaterMonoisotopicMass;
foreach (Modification mod in relevantDatabaseMod.Value)
{
double modifiedMass = basePrecursorMass + mod.MonoisotopicMass.Value;
if (modifiedMass <= MaxFragmentSize) //if the precursor is larger than the index allows, then don't add it
{
int precursorBin = (int)Math.Round(modifiedMass * FragmentBinsPerDalton);
if (precursorIndex[precursorBin] == null)
{
precursorIndex[precursorBin] = new List <int> {
peptideId
};
}
else
{
precursorIndex[precursorBin].Add(peptideId);
}
}
}
}
}
protected override MyTaskResults RunSpecific(string OutputFolder, List <DbForTask> dbFilenameList, List <string> currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList)
{
if (SearchParameters.DoQuantification)
{
// disable quantification if a .mgf is being used
if (currentRawFileList.Any(x => Path.GetExtension(x).Equals(".mgf", StringComparison.OrdinalIgnoreCase)))
{
SearchParameters.DoQuantification = false;
}
//if we're doing SILAC, assign and add the silac labels to the residue dictionary
else if (SearchParameters.SilacLabels != null || SearchParameters.StartTurnoverLabel != null || SearchParameters.EndTurnoverLabel != null)
{
char heavyLabel = 'a'; //char to assign
//add the Turnoverlabels to the silacLabels list. They weren't there before just to prevent duplication in the tomls
if (SearchParameters.StartTurnoverLabel != null || SearchParameters.EndTurnoverLabel != null)
{
//original silacLabels object is null, so we need to initialize it
SearchParameters.SilacLabels = new List <SilacLabel>();
if (SearchParameters.StartTurnoverLabel != null)
{
var updatedLabel = SilacConversions.UpdateAminoAcidLabel(SearchParameters.StartTurnoverLabel, heavyLabel);
heavyLabel = updatedLabel.nextHeavyLabel;
SearchParameters.StartTurnoverLabel = updatedLabel.updatedLabel;
SearchParameters.SilacLabels.Add(SearchParameters.StartTurnoverLabel);
}
if (SearchParameters.EndTurnoverLabel != null)
{
var updatedLabel = SilacConversions.UpdateAminoAcidLabel(SearchParameters.EndTurnoverLabel, heavyLabel);
heavyLabel = updatedLabel.nextHeavyLabel;
SearchParameters.EndTurnoverLabel = updatedLabel.updatedLabel;
SearchParameters.SilacLabels.Add(SearchParameters.EndTurnoverLabel);
}
}
else
{
//change the silac residues to lower case amino acids (currently null)
List <SilacLabel> updatedLabels = new List <SilacLabel>();
for (int i = 0; i < SearchParameters.SilacLabels.Count; i++)
{
var updatedLabel = SilacConversions.UpdateAminoAcidLabel(SearchParameters.SilacLabels[i], heavyLabel);
heavyLabel = updatedLabel.nextHeavyLabel;
updatedLabels.Add(updatedLabel.updatedLabel);
}
SearchParameters.SilacLabels = updatedLabels;
}
}
}
//if no quant, remove any silac labels that may have been added, because they screw up downstream analysis
if (!SearchParameters.DoQuantification) //using "if" instead of "else", because DoQuantification can change if it's an mgf
{
SearchParameters.SilacLabels = null;
}
LoadModifications(taskId, out var variableModifications, out var fixedModifications, out var localizeableModificationTypes);
// load proteins
List <Protein> proteinList = LoadProteins(taskId, dbFilenameList, SearchParameters.SearchTarget, SearchParameters.DecoyType, localizeableModificationTypes, CommonParameters);
SanitizeProteinDatabase(proteinList, SearchParameters.TCAmbiguity);
// load spectral libraries
var spectralLibrary = LoadSpectralLibraries(taskId, dbFilenameList);
// write prose settings
ProseCreatedWhileRunning.Append("The following search settings were used: ");
ProseCreatedWhileRunning.Append("protease = " + CommonParameters.DigestionParams.Protease + "; ");
ProseCreatedWhileRunning.Append("maximum missed cleavages = " + CommonParameters.DigestionParams.MaxMissedCleavages + "; ");
ProseCreatedWhileRunning.Append("minimum peptide length = " + CommonParameters.DigestionParams.MinPeptideLength + "; ");
ProseCreatedWhileRunning.Append(CommonParameters.DigestionParams.MaxPeptideLength == int.MaxValue ?
"maximum peptide length = unspecified; " :
"maximum peptide length = " + CommonParameters.DigestionParams.MaxPeptideLength + "; ");
ProseCreatedWhileRunning.Append("initiator methionine behavior = " + CommonParameters.DigestionParams.InitiatorMethionineBehavior + "; ");
ProseCreatedWhileRunning.Append("fixed modifications = " + string.Join(", ", fixedModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("variable modifications = " + string.Join(", ", variableModifications.Select(m => m.IdWithMotif)) + "; ");
ProseCreatedWhileRunning.Append("max mods per peptide = " + CommonParameters.DigestionParams.MaxModsForPeptide + "; ");
ProseCreatedWhileRunning.Append("max modification isoforms = " + CommonParameters.DigestionParams.MaxModificationIsoforms + "; ");
ProseCreatedWhileRunning.Append("precursor mass tolerance = " + CommonParameters.PrecursorMassTolerance + "; ");
ProseCreatedWhileRunning.Append("product mass tolerance = " + CommonParameters.ProductMassTolerance + "; ");
ProseCreatedWhileRunning.Append("report PSM ambiguity = " + CommonParameters.ReportAllAmbiguity + ". ");
ProseCreatedWhileRunning.Append("The combined search database contained " + proteinList.Count(p => !p.IsDecoy)
+ " non-decoy protein entries including " + proteinList.Count(p => p.IsContaminant) + " contaminant sequences. ");
// start the search task
MyTaskResults = new MyTaskResults(this);
List <PeptideSpectralMatch> allPsms = new List <PeptideSpectralMatch>();
//generate an array to store category specific fdr values (for speedy semi/nonspecific searches)
int numFdrCategories = (int)(Enum.GetValues(typeof(FdrCategory)).Cast <FdrCategory>().Last() + 1); //+1 because it starts at zero
List <PeptideSpectralMatch>[] allCategorySpecificPsms = new List <PeptideSpectralMatch> [numFdrCategories];
for (int i = 0; i < numFdrCategories; i++)
{
allCategorySpecificPsms[i] = new List <PeptideSpectralMatch>();
}
FlashLfqResults flashLfqResults = null;
MyFileManager myFileManager = new MyFileManager(SearchParameters.DisposeOfFileWhenDone);
var fileSpecificCommonParams = fileSettingsList.Select(b => SetAllFileSpecificCommonParams(CommonParameters, b));
int completedFiles = 0;
object indexLock = new object();
object psmLock = new object();
Status("Searching files...", taskId);
Status("Searching files...", new List <string> {
taskId, "Individual Spectra Files"
});
Dictionary <string, int[]> numMs2SpectraPerFile = new Dictionary <string, int[]>();
for (int spectraFileIndex = 0; spectraFileIndex < currentRawFileList.Count; spectraFileIndex++)
{
if (GlobalVariables.StopLoops)
{
break;
}
var origDataFile = currentRawFileList[spectraFileIndex];
// mark the file as in-progress
StartingDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
CommonParameters combinedParams = SetAllFileSpecificCommonParams(CommonParameters, fileSettingsList[spectraFileIndex]);
MassDiffAcceptor massDiffAcceptor = GetMassDiffAcceptor(combinedParams.PrecursorMassTolerance, SearchParameters.MassDiffAcceptorType, SearchParameters.CustomMdac);
var thisId = new List <string> {
taskId, "Individual Spectra Files", origDataFile
};
NewCollection(Path.GetFileName(origDataFile), thisId);
Status("Loading spectra file...", thisId);
MsDataFile myMsDataFile = myFileManager.LoadFile(origDataFile, combinedParams);
Status("Getting ms2 scans...", thisId);
Ms2ScanWithSpecificMass[] arrayOfMs2ScansSortedByMass = GetMs2Scans(myMsDataFile, origDataFile, combinedParams).OrderBy(b => b.PrecursorMass).ToArray();
numMs2SpectraPerFile.Add(Path.GetFileNameWithoutExtension(origDataFile), new int[] { myMsDataFile.GetAllScansList().Count(p => p.MsnOrder == 2), arrayOfMs2ScansSortedByMass.Length });
myFileManager.DoneWithFile(origDataFile);
PeptideSpectralMatch[] fileSpecificPsms = new PeptideSpectralMatch[arrayOfMs2ScansSortedByMass.Length];
// modern search
if (SearchParameters.SearchType == SearchType.Modern)
{
for (int currentPartition = 0; currentPartition < combinedParams.TotalPartitions; currentPartition++)
{
List <PeptideWithSetModifications> peptideIndex = null;
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count / combinedParams.TotalPartitions,
((currentPartition + 1) * proteinList.Count / combinedParams.TotalPartitions) - (currentPartition * proteinList.Count / combinedParams.TotalPartitions));
Status("Getting fragment dictionary...", new List <string> {
taskId
});
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, SearchParameters.SilacLabels,
SearchParameters.StartTurnoverLabel, SearchParameters.EndTurnoverLabel, currentPartition, SearchParameters.DecoyType, combinedParams, FileSpecificParameters,
SearchParameters.MaxFragmentSize, false, dbFilenameList.Select(p => new FileInfo(p.FilePath)).ToList(), SearchParameters.TCAmbiguity, new List <string> {
taskId
});
List <int>[] fragmentIndex = null;
List <int>[] precursorIndex = null;
lock (indexLock)
{
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, ref precursorIndex, proteinList, taskId);
}
Status("Searching files...", taskId);
new ModernSearchEngine(fileSpecificPsms, arrayOfMs2ScansSortedByMass, peptideIndex, fragmentIndex, currentPartition,
combinedParams, this.FileSpecificParameters, massDiffAcceptor, SearchParameters.MaximumMassThatFragmentIonScoreIsDoubled, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + combinedParams.TotalPartitions + "!", thisId));
if (GlobalVariables.StopLoops)
{
break;
}
}
}
// nonspecific search
else if (SearchParameters.SearchType == SearchType.NonSpecific)
{
PeptideSpectralMatch[][] fileSpecificPsmsSeparatedByFdrCategory = new PeptideSpectralMatch[numFdrCategories][]; //generate an array of all possible locals
for (int i = 0; i < numFdrCategories; i++) //only add if we're using for FDR, else ignore it as null.
{
fileSpecificPsmsSeparatedByFdrCategory[i] = new PeptideSpectralMatch[arrayOfMs2ScansSortedByMass.Length];
}
//create params for N, C, or both if semi
List <CommonParameters> paramsToUse = new List <CommonParameters> {
combinedParams
};
if (combinedParams.DigestionParams.SearchModeType == CleavageSpecificity.Semi) //if semi, we need to do both N and C to hit everything
{
paramsToUse.Clear();
List <FragmentationTerminus> terminiToUse = new List <FragmentationTerminus> {
FragmentationTerminus.N, FragmentationTerminus.C
};
foreach (FragmentationTerminus terminus in terminiToUse) //set both termini
{
paramsToUse.Add(combinedParams.CloneWithNewTerminus(terminus));
}
}
//Compress array of deconvoluted ms2 scans to avoid searching the same ms2 multiple times while still identifying coisolated peptides
List <int>[] coisolationIndex = new List <int>[] { new List <int>() };
if (arrayOfMs2ScansSortedByMass.Length != 0)
{
int maxScanNumber = arrayOfMs2ScansSortedByMass.Max(x => x.OneBasedScanNumber);
coisolationIndex = new List <int> [maxScanNumber + 1];
for (int i = 0; i < arrayOfMs2ScansSortedByMass.Length; i++)
{
int scanNumber = arrayOfMs2ScansSortedByMass[i].OneBasedScanNumber;
if (coisolationIndex[scanNumber] == null)
{
coisolationIndex[scanNumber] = new List <int> {
i
};
}
else
{
coisolationIndex[scanNumber].Add(i);
}
}
coisolationIndex = coisolationIndex.Where(x => x != null).ToArray();
}
//foreach terminus we're going to look at
foreach (CommonParameters paramToUse in paramsToUse)
{
//foreach database partition
for (int currentPartition = 0; currentPartition < paramToUse.TotalPartitions; currentPartition++)
{
List <PeptideWithSetModifications> peptideIndex = null;
List <Protein> proteinListSubset = proteinList.GetRange(currentPartition * proteinList.Count / paramToUse.TotalPartitions,
((currentPartition + 1) * proteinList.Count / paramToUse.TotalPartitions) - (currentPartition * proteinList.Count / paramToUse.TotalPartitions));
List <int>[] fragmentIndex = null;
List <int>[] precursorIndex = null;
Status("Getting fragment dictionary...", new List <string> {
taskId
});
var indexEngine = new IndexingEngine(proteinListSubset, variableModifications, fixedModifications, SearchParameters.SilacLabels,
SearchParameters.StartTurnoverLabel, SearchParameters.EndTurnoverLabel, currentPartition, SearchParameters.DecoyType, paramToUse, FileSpecificParameters,
SearchParameters.MaxFragmentSize, true, dbFilenameList.Select(p => new FileInfo(p.FilePath)).ToList(), SearchParameters.TCAmbiguity, new List <string> {
taskId
});
lock (indexLock)
{
GenerateIndexes(indexEngine, dbFilenameList, ref peptideIndex, ref fragmentIndex, ref precursorIndex, proteinList, taskId);
}
Status("Searching files...", taskId);
new NonSpecificEnzymeSearchEngine(fileSpecificPsmsSeparatedByFdrCategory, arrayOfMs2ScansSortedByMass, coisolationIndex, peptideIndex, fragmentIndex,
precursorIndex, currentPartition, paramToUse, this.FileSpecificParameters, variableModifications, massDiffAcceptor,
SearchParameters.MaximumMassThatFragmentIonScoreIsDoubled, thisId).Run();
ReportProgress(new ProgressEventArgs(100, "Done with search " + (currentPartition + 1) + "/" + paramToUse.TotalPartitions + "!", thisId));
if (GlobalVariables.StopLoops)
{
break;
}
}
}
lock (psmLock)
{
for (int i = 0; i < allCategorySpecificPsms.Length; i++)
{
if (allCategorySpecificPsms[i] != null)
{
allCategorySpecificPsms[i].AddRange(fileSpecificPsmsSeparatedByFdrCategory[i]);
}
}
}
}
// classic search
else
{
Status("Starting search...", thisId);
var newClassicSearchEngine = new ClassicSearchEngine(fileSpecificPsms, arrayOfMs2ScansSortedByMass, variableModifications, fixedModifications, SearchParameters.SilacLabels,
SearchParameters.StartTurnoverLabel, SearchParameters.EndTurnoverLabel, proteinList, massDiffAcceptor, combinedParams, this.FileSpecificParameters, spectralLibrary, thisId, SearchParameters.WriteSpectralLibrary);
newClassicSearchEngine.Run();
ReportProgress(new ProgressEventArgs(100, "Done with search!", thisId));
}
//look for internal fragments
if (SearchParameters.MinAllowedInternalFragmentLength != 0)
{
MatchInternalFragmentIons(fileSpecificPsms, arrayOfMs2ScansSortedByMass, combinedParams, SearchParameters.MinAllowedInternalFragmentLength);
}
// calculate/set spectral angles if there is a spectral library being used
if (spectralLibrary != null)
{
Status("Calculating spectral library similarity...", thisId);
}
SpectralLibrarySearchFunction.CalculateSpectralAngles(spectralLibrary, fileSpecificPsms, arrayOfMs2ScansSortedByMass, combinedParams);
lock (psmLock)
{
allPsms.AddRange(fileSpecificPsms);
}
completedFiles++;
FinishedDataFile(origDataFile, new List <string> {
taskId, "Individual Spectra Files", origDataFile
});
ReportProgress(new ProgressEventArgs(completedFiles / currentRawFileList.Count, "Searching...", new List <string> {
taskId, "Individual Spectra Files"
}));
}
if (spectralLibrary != null)
{
spectralLibrary.CloseConnections();
}
ReportProgress(new ProgressEventArgs(100, "Done with all searches!", new List <string> {
taskId, "Individual Spectra Files"
}));
int numNotches = GetNumNotches(SearchParameters.MassDiffAcceptorType, SearchParameters.CustomMdac);
//resolve category specific fdrs (for speedy semi and nonspecific
if (SearchParameters.SearchType == SearchType.NonSpecific)
{
allPsms = NonSpecificEnzymeSearchEngine.ResolveFdrCategorySpecificPsms(allCategorySpecificPsms, numNotches, taskId, CommonParameters, FileSpecificParameters);
}
PostSearchAnalysisParameters parameters = new PostSearchAnalysisParameters
{
SearchTaskResults = MyTaskResults,
SearchTaskId = taskId,
SearchParameters = SearchParameters,
ProteinList = proteinList,
AllPsms = allPsms,
VariableModifications = variableModifications,
FixedModifications = fixedModifications,
ListOfDigestionParams = new HashSet <DigestionParams>(fileSpecificCommonParams.Select(p => p.DigestionParams)),
CurrentRawFileList = currentRawFileList,
MyFileManager = myFileManager,
NumNotches = numNotches,
OutputFolder = OutputFolder,
IndividualResultsOutputFolder = Path.Combine(OutputFolder, "Individual File Results"),
FlashLfqResults = flashLfqResults,
FileSettingsList = fileSettingsList,
NumMs2SpectraPerFile = numMs2SpectraPerFile,
DatabaseFilenameList = dbFilenameList
};
PostSearchAnalysisTask postProcessing = new PostSearchAnalysisTask
{
Parameters = parameters,
FileSpecificParameters = this.FileSpecificParameters,
CommonParameters = CommonParameters
};
return(postProcessing.Run());
}
protected override MetaMorpheusEngineResults RunSpecific()
{
double progress = 0;
int oldPercentProgress = 0;
// digest database
List <PeptideWithSetModifications> globalPeptides = new List <PeptideWithSetModifications>();
int maxThreadsPerFile = CommonParameters.MaxThreadsToUsePerFile;
int[] threads = Enumerable.Range(0, maxThreadsPerFile).ToArray();
Parallel.ForEach(threads, (i) =>
{
List <PeptideWithSetModifications> localPeptides = new List <PeptideWithSetModifications>();
for (; i < ProteinList.Count; i += maxThreadsPerFile)
{
// Stop loop if canceled
if (GlobalVariables.StopLoops)
{
return;
}
localPeptides.AddRange(ProteinList[i].Digest(CommonParameters.DigestionParams, FixedModifications, VariableModifications, SilacLabels));
progress++;
var percentProgress = (int)((progress / ProteinList.Count) * 100);
if (percentProgress > oldPercentProgress)
{
oldPercentProgress = percentProgress;
ReportProgress(new ProgressEventArgs(percentProgress, "Digesting proteins...", NestedIds));
}
}
lock (globalPeptides)
{
globalPeptides.AddRange(localPeptides);
}
});
// sort peptides by mass
List <PeptideWithSetModifications> peptidesSortedByMass = globalPeptides.OrderBy(p => p.MonoisotopicMass).ToList();
globalPeptides = null; //RAM
//create precursor index (if specified)
List <int>[] precursorIndex = null;
if (GeneratePrecursorIndex)
{
precursorIndex = CreateNewPrecursorIndex(peptidesSortedByMass);
}
bool addInteriorTerminalModsToPrecursorIndex = GeneratePrecursorIndex && CommonParameters.DigestionParams.Protease.Name.Contains("single");
List <Modification> terminalModifications = addInteriorTerminalModsToPrecursorIndex ?
NonSpecificEnzymeSearchEngine.GetVariableTerminalMods(CommonParameters.DigestionParams.FragmentationTerminus, VariableModifications) :
null;
// create fragment index
List <int>[] fragmentIndex;
try
{
fragmentIndex = new List <int> [(int)Math.Ceiling(MaxFragmentSize) * FragmentBinsPerDalton + 1];
}
catch (OutOfMemoryException)
{
throw new MetaMorpheusException("Max fragment mass too large for indexing engine; try \"Classic Search\" mode, or make the maximum fragment mass smaller");
}
// populate fragment index
progress = 0;
oldPercentProgress = 0;
for (int peptideId = 0; peptideId < peptidesSortedByMass.Count; peptideId++)
{
List <Product> fragments = peptidesSortedByMass[peptideId].Fragment(CommonParameters.DissociationType, CommonParameters.DigestionParams.FragmentationTerminus).ToList();
List <double> fragmentMasses = fragments.Select(m => m.NeutralMass).ToList();
foreach (double theoreticalFragmentMass in fragmentMasses)
{
double tfm = theoreticalFragmentMass;
//if low res round
if (CommonParameters.DissociationType == MassSpectrometry.DissociationType.LowCID)
{
tfm = Math.Round(theoreticalFragmentMass / 1.0005079, 0) * 1.0005079;
}
if (tfm < MaxFragmentSize && tfm > 0)
{
int fragmentBin = (int)Math.Round(tfm * FragmentBinsPerDalton);
if (fragmentIndex[fragmentBin] == null)
{
fragmentIndex[fragmentBin] = new List <int> {
peptideId
};
}
else
{
fragmentIndex[fragmentBin].Add(peptideId);
}
}
}
//Add terminal mods if needed (do it here rather than earlier so that we don't have to fragment twice
if (addInteriorTerminalModsToPrecursorIndex)
{
AddInteriorTerminalModsToPrecursorIndex(precursorIndex, fragments, peptidesSortedByMass[peptideId], peptideId, terminalModifications);
}
progress++;
var percentProgress = (int)((progress / peptidesSortedByMass.Count) * 100);
if (percentProgress > oldPercentProgress)
{
oldPercentProgress = percentProgress;
ReportProgress(new ProgressEventArgs(percentProgress, "Fragmenting peptides...", NestedIds));
}
}
return(new IndexingResults(peptidesSortedByMass, fragmentIndex, precursorIndex, this));
}
