internal void LoadFeatures(List <DatasetInformationViewModel> workFlowDatasets = null, IProgress <ProgressData> workflowProgress = null)
{
var featureCache = new FeatureLoader {
Providers = this.analysis.DataProviders
};
this.ShouldShowProgress = true;
var selectedFiles = workFlowDatasets ?? this.Datasets.Where(file => !file.DoingWork).Where(ds => ds.IsSelected).ToList();
foreach (var file in selectedFiles)
{
file.DatasetState = DatasetInformationViewModel.DatasetStates.FindingFeatures;
ThreadSafeDispatcher.Invoke(() => this.PlotMsFeaturesCommand.RaiseCanExecuteChanged());
ThreadSafeDispatcher.Invoke(() => this.FindMsFeaturesCommand.RaiseCanExecuteChanged());
}
var taskBarProgress = TaskBarProgress.GetInstance();
taskBarProgress.ShowProgress(this, true);
workflowProgress = workflowProgress ?? new Progress <ProgressData>();
IProgress <ProgressData> totalProgressRpt = new Progress <ProgressData>(pd =>
{
this.TotalProgress = pd.Percent;
taskBarProgress.SetProgress(this, pd.Percent);
workflowProgress.Report(pd);
});
var totalProgressData = new ProgressData(totalProgressRpt);
DatabaseIndexer.IndexClustersDrop(NHibernateUtil.Path);
DatabaseIndexer.IndexFeaturesDrop(NHibernateUtil.Path);
var i = 1;
foreach (var file in selectedFiles)
{
// Set range based on file
totalProgressData.StepRange((i++ *100.0) / selectedFiles.Count);
var fileInstance = file;
var progData = new ProgressData(new Progress <ProgressData>(pd =>
{
fileInstance.Progress = pd.Percent;
// Report file progress
totalProgressData.Report(fileInstance.Progress);
}));
var progressRpt = new Progress <ProgressData>(pd => progData.Report(pd.Percent));
progData.StepRange(30);
IList <UMCLight> features;
// Load features from the database.
try
{
this.analysis.DataProviders.DatabaseLock.EnterReadLock();
features = featureCache.LoadDataset(
file.Dataset,
this.analysis.Options.MsFilteringOptions,
this.analysis.Options.LcmsFindingOptions,
this.analysis.Options.LcmsFilteringOptions,
this.analysis.Options.DataLoadOptions,
this.analysis.DataProviders.ScanSummaryProviderCache,
this.analysis.DataProviders.IdentificationProviderCache,
progressRpt);
}
finally
{ // Always close read lock, even during failure condition so we don't have a recursive lock error.
this.analysis.DataProviders.DatabaseLock.ExitReadLock();
}
if (!this.featuresByDataset.ContainsKey(file.Dataset))
{
this.featuresByDataset.Add(file.Dataset, new List <UMCLight>());
}
this.featuresByDataset[file.Dataset] = features;
file.DatasetState = DatasetInformationViewModel.DatasetStates.PersistingFeatures;
ThreadSafeDispatcher.Invoke(() => this.PlotMsFeaturesCommand.RaiseCanExecuteChanged());
// TODO: We were using this log file to track speed changes for writing the database. We probably don't need it anymore.
using (var logger = new StreamWriter("nhibernate_stats.txt", true))
{
logger.WriteLine();
var stopWatch = new Stopwatch();
stopWatch.Start();
var scanSumProvider =
this.analysis.DataProviders.ScanSummaryProviderCache.GetScanSummaryProvider(
file.Dataset.DatasetId);
if (scanSumProvider.IsBackedByFile)
{
var ssDao = this.analysis.DataProviders.ScanSummaryDao;
ssDao.DeleteByDatasetId(file.Dataset.DatasetId);
// Add all of the Scan Summaries for this dataset to the database, but first properly set the dataset ID
ssDao.AddAllStateless(
scanSumProvider.GetScanSummaries().Select(
summ =>
{
summ.DatasetId = file.Dataset.DatasetId;
return(summ);
}).ToList());
}
progData.StepRange(100);
// Cache features to database.
try
{
this.analysis.DataProviders.DatabaseLock.EnterWriteLock();
featureCache.CacheFeatures(features, progressRpt);
}
catch (NonUniqueObjectException ex)
{
MessageBox.Show("Could not completely persist features: " + ex.Message);
}
catch (Exception ex) // TODO: Figure out which exception should actually be caught here
{
MessageBox.Show("Could not persist features to database: " + ex.Message);
file.DatasetState = DatasetInformationViewModel.DatasetStates.Loaded;
continue;
}
finally
{ // Always close write lock, even during failure condition so we don't have a recursive lock error.
this.analysis.DataProviders.DatabaseLock.ExitWriteLock();
}
stopWatch.Stop();
logger.WriteLine("Writing: {0}s", stopWatch.Elapsed.TotalSeconds);
}
file.DatasetState = DatasetInformationViewModel.DatasetStates.FeaturesFound;
ThreadSafeDispatcher.Invoke(() => this.FindMsFeaturesCommand.RaiseCanExecuteChanged());
file.Progress = 0;
}
DatabaseIndexer.IndexFeatures(NHibernateUtil.Path);
taskBarProgress.ShowProgress(this, false);
this.ShouldShowProgress = false;
}
internal void ClusterFeatures(IProgress <ProgressData> workflowProgress = null)
{
var taskBarProgress = TaskBarProgress.GetInstance();
taskBarProgress.ShowProgress(this, true);
workflowProgress = workflowProgress ?? new Progress <ProgressData>();
IProgress <ProgressData> internalProgress = new Progress <ProgressData>(pd =>
{
this.progress.Report((int)pd.Percent);
this.ProgressPercent = pd.Percent;
taskBarProgress.SetProgress(this, pd.Percent);
workflowProgress.Report(pd);
});
this.algorithms = this.builder.GetAlgorithmProvider(this.options);
var clusterer = this.algorithms.Clusterer;
clusterer.Parameters = LcmsClusteringOptions.ConvertToOmics(this.options.LcmsClusteringOptions);
this.featureCache = this.analysis.DataProviders.FeatureCache;
if (clusterer is PromexClusterer)
{
var promexClusterer = clusterer as PromexClusterer;
promexClusterer.Readers = this.analysis.DataProviders.ScanSummaryProviderCache;
}
foreach (var dataset in this.Datasets)
{
if (dataset.FeaturesFound)
{
dataset.DatasetState = DatasetInformationViewModel.DatasetStates.Clustering;
}
}
ThreadSafeDispatcher.Invoke(this.ClusterFeaturesCommand.RaiseCanExecuteChanged);
ThreadSafeDispatcher.Invoke(this.DisplayClustersCommand.RaiseCanExecuteChanged);
this.ShouldShowProgress = true;
var progData = new ProgressData(internalProgress);
var clusterProgress = new Progress <ProgressData>(pd => progData.Report(pd.Percent));
this.analysis.DataProviders.DatabaseLock.EnterWriteLock();
DatabaseIndexer.IndexClustersDrop(NHibernateUtil.Path);
this.analysis.DataProviders.ClusterCache.ClearAllClusters();
this.analysis.DataProviders.DatabaseLock.ExitWriteLock();
// The id for a cluster - keep track here to avoid duplicates when separating by charge.
var clusterCount = 0;
// Here we see if we need to separate the charge...
// IMS is said to require charge separation
if (!this.analysis.Options.LcmsClusteringOptions.ShouldSeparateCharge)
{
progData.StepRange(45);
var features = new List <UMCLight>();
var i = 0;
var datasets = this.Datasets.Where(ds => ds.FeaturesFound).ToList();
foreach (var dataset in datasets)
{
this.analysis.DataProviders.DatabaseLock.EnterReadLock();
features.AddRange(this.featureCache.FindByDatasetId(dataset.DatasetId));
this.analysis.DataProviders.DatabaseLock.ExitReadLock();
progData.Report(++i, datasets.Count);
}
progData.StepRange(100);
ClusterGroupOfFeatures(clusterer, features, ref clusterCount, clusterProgress);
}
else
{
var maxChargeState = this.featureCache.FindMaxCharge();
// Here we cluster all charge states separately. Probably IMS Data.
for (var chargeState = 1; chargeState <= maxChargeState; chargeState++)
{
var maxPercent = ((100.0 * chargeState) / maxChargeState);
// TODO: Add restriction by selected dataset ids?
var features = this.featureCache.FindByCharge(chargeState);
if (features.Count < 1)
{
continue;
}
progData.StepRange(maxPercent);
ClusterGroupOfFeatures(clusterer, features, ref clusterCount, clusterProgress);
}
this.analysis.Clusters = this.analysis.DataProviders.ClusterCache.FindAll();
}
this.analysis.DataProviders.DatabaseLock.EnterWriteLock();
DatabaseIndexer.IndexClusters(NHibernateUtil.Path);
this.analysis.DataProviders.DatabaseLock.ExitWriteLock();
foreach (var dataset in this.Datasets)
{
if (dataset.DatasetState == DatasetInformationViewModel.DatasetStates.PersistingClusters)
{
dataset.DatasetState = DatasetInformationViewModel.DatasetStates.Clustered;
}
}
try
{
// Write to file
this.WriteClusterData(string.Format("{0}_crosstab.tsv", this.analysis.AnalysisName), this.analysis.Clusters);
}
catch (Exception ex)
{
var errMsg = "Error writing results to text file: " + ex.Message;
Logger.PrintMessage(errMsg);
// Todo: Add this: if (!GlobalSettings.AutomatedAnalysisMode)
MessageBox.Show(errMsg);
}
ThreadSafeDispatcher.Invoke(this.ClusterFeaturesCommand.RaiseCanExecuteChanged);
ThreadSafeDispatcher.Invoke(this.DisplayClustersCommand.RaiseCanExecuteChanged);
taskBarProgress.ShowProgress(this, false);
this.ShouldShowProgress = false;
}
/// <summary>
/// Perform alignment.
/// </summary>
/// <param name="workFlowDatasets">Datasets to run on when being called externally form this view model.</param>
/// <param name="workflowProgress">The progress reporter for when this method is called externally from this view model.</param>
internal void AlignToBaseline(List <DatasetInformationViewModel> workFlowDatasets = null, IProgress <ProgressData> workflowProgress = null)
{
// Use Promiscuous points when aligning to an AMT tag database
// Do not use Promiscuous points when aligning to a baseline dataset
this.analysis.Options.AlignmentOptions.UsePromiscuousPoints = !this.ShouldAlignToBaseline;
// Flag whether we are aligning to an AMT tag database
this.analysis.Options.AlignmentOptions.LCMSWarpOptions.AlignToMassTagDatabase = !this.ShouldAlignToBaseline;
// Show the progress bar
this.ShowAlignmentProgress = true;
var taskBarProgress = TaskBarProgress.GetInstance();
taskBarProgress.ShowProgress(this, true);
// Update algorithms and providers
var featureCache = new FeatureLoader {
Providers = this.analysis.DataProviders
};
this.algorithms = this.builder.GetAlgorithmProvider(this.analysis.Options);
////this.algorithms.DatabaseAligner.Progress += aligner_Progress;
////this.algorithms.DatasetAligner.Progress += aligner_Progress;
this.aligner.m_algorithms = this.algorithms;
var baselineFeatures = new List <UMCLight>();
if (this.ShouldAlignToBaseline)
{
baselineFeatures = featureCache.Providers.FeatureCache.FindByDatasetId(this.selectedBaseline.DatasetId);
this.SelectedBaseline.DatasetState = DatasetInformationViewModel.DatasetStates.Baseline;
var priorAlignment = (from x in this.alignmentInformation where x.DatasetID == this.selectedBaseline.DatasetId select x).ToList();
if (priorAlignment.Any())
{
this.alignmentInformation.Remove(priorAlignment.Single());
}
}
var alignmentData = new AlignmentDAOHibernate();
alignmentData.ClearAll();
var selectedFiles = workFlowDatasets ??
this.Datasets.Where(file => file.IsSelected && !file.DoingWork &&
(this.ShouldAlignToAMT || !file.IsBaseline)).ToList();
foreach (var file in selectedFiles)
{
file.DatasetState = DatasetInformationViewModel.DatasetStates.Aligning;
}
workflowProgress = workflowProgress ?? new Progress <ProgressData>();
IProgress <ProgressData> totalProgress = new Progress <ProgressData>(pd =>
{
this.AlignmentProgress = pd.Percent;
workflowProgress.Report(pd);
taskBarProgress.SetProgress(this, pd.Percent);
});
var totalProgressData = new ProgressData(totalProgress);
DatabaseIndexer.IndexClustersDrop(NHibernateUtil.Path);
DatabaseIndexer.IndexFeaturesDrop(NHibernateUtil.Path);
var i = 1;
foreach (var file in selectedFiles)
{
ThreadSafeDispatcher.Invoke(() => this.AlignCommand.RaiseCanExecuteChanged());
ThreadSafeDispatcher.Invoke(() => this.DisplayAlignmentCommand.RaiseCanExecuteChanged());
ThreadSafeDispatcher.Invoke(() => this.SaveAlignmentPlotsCommand.RaiseCanExecuteChanged());
if ((file.Dataset.IsBaseline || !file.FeaturesFound) && this.ShouldAlignToBaseline)
{
file.DatasetState = DatasetInformationViewModel.DatasetStates.Aligned;
continue;
}
this.analysis.DataProviders.DatabaseLock.EnterReadLock();
IList <UMCLight> features = featureCache.Providers.FeatureCache.FindByDatasetId(file.DatasetId);
this.analysis.DataProviders.DatabaseLock.ExitReadLock();
AlignmentData alignment;
totalProgressData.StepRange((100.0 * i++) / selectedFiles.Count);
var fileInstance = file;
var datasetProgress =
new Progress <ProgressData>(
pd =>
{
fileInstance.Progress = pd.Percent;
totalProgressData.Report(pd.Percent);
});
if (this.ShouldAlignToBaseline)
{
// Aligning to a baseline dataset
alignment = this.aligner.AlignToDataset(ref features, file.Dataset, baselineFeatures, datasetProgress);
alignment.BaselineIsAmtDB = false;
}
else
{
// Aligning to a database
alignment = this.aligner.AlignToDatabase(ref features, file.Dataset, this.analysis.MassTagDatabase, datasetProgress);
alignment.BaselineIsAmtDB = true;
}
// Check if there is information from a previous alignment for this dataset. If so, replace it. If not, just add the new one.
var priorAlignment = this.alignmentInformation.Where(x => x.DatasetID == alignment.DatasetID).ToList();
if (priorAlignment.Any())
{
this.alignmentInformation.Remove(priorAlignment.Single());
this.alignmentInformation.Add(alignment);
}
else
{
this.alignmentInformation.Add(alignment);
}
file.Dataset.AlignmentData = alignment;
this.analysis.DataProviders.DatabaseLock.EnterWriteLock();
featureCache.CacheFeatures(features);
this.analysis.DataProviders.DatabaseLock.ExitWriteLock();
file.DatasetState = DatasetInformationViewModel.DatasetStates.Aligned;
ThreadSafeDispatcher.Invoke(() => this.AlignCommand.RaiseCanExecuteChanged());
ThreadSafeDispatcher.Invoke(() => this.DisplayAlignmentCommand.RaiseCanExecuteChanged());
ThreadSafeDispatcher.Invoke(() => this.SaveAlignmentPlotsCommand.RaiseCanExecuteChanged());
file.Progress = 0;
}
if (this.ShouldAlignToBaseline)
{
this.SelectedBaseline.DatasetState = DatasetInformationViewModel.DatasetStates.Aligned;
}
DatabaseIndexer.IndexFeatures(NHibernateUtil.Path);
taskBarProgress.ShowProgress(this, false);
this.ShowAlignmentProgress = false;
this.AlignmentProgress = 0;
}
public void CreateUMCClusterLight(string databasePath, bool indexDatabase)
{
// If the database is not index then do so...but before the session to the db is opened.
if (indexDatabase)
{
DatabaseIndexer.IndexClusters(databasePath);
DatabaseIndexer.IndexFeatures(databasePath);
}
// This is a factory based method that creates a set of data access providers used throughout MultiAlign
var providers = DataAccessFactory.CreateDataAccessProviders(databasePath, false);
// If you just wanted the clusters you could do this:
// 1. Connect to the database
//NHibernateUtil.ConnectToDatabase(databasePath, false);
// 2. Then extract all of the clusters
//IUmcClusterDAO clusterCache = new UmcClusterDAOHibernate();
//List<UMCClusterLight> clusters = clusterCache.FindAll();
var clusters = providers.ClusterCache.FindAll();
var shouldGetMsFeatures = true;
var shouldGetMsMsFeatures = true;
var shouldGetRawData = false;
// This gets all of the dataset information and maps to a dictionary...if you want the raw data
// otherwise comment this out.
var datasets = providers.DatasetCache.FindAll();
var datasetMap = new Dictionary <int, DatasetInformation>();
datasets.ForEach(x => datasetMap.Add(x.DatasetId, x));
foreach (var cluster in clusters)
{
cluster.ReconstructUMCCluster(providers,
true,
false,
shouldGetMsFeatures,
shouldGetMsMsFeatures);
foreach (var feature in cluster.Features)
{
foreach (var msFeature in feature.Features)
{
foreach (var spectrumMetaData in msFeature.MSnSpectra)
{
// then you can do stuff with the ms/ms spectra
// If you had the path to the raw file, you could create a reader for you to extract the MS/MS spectra
// This supports mzXML and .RAW Thermo files based on the file extension.
if (shouldGetRawData)
{
DatasetInformation info = null;
var hasKey = datasetMap.TryGetValue(spectrumMetaData.GroupId, out info);
if (hasKey)
{
if (info.RawFile != null)
{
// This might seem kind of klunky, but it's called a bridge, this way I can access
// MS/MS spectra from PNNLOmics without having to reference any of the Thermo DLL's
// Nor support file reading capability. This is also nice because I don't have to load
// several MS/MS spectra when analyzing large datasets for my spectral clustering work.
var rawReader = new InformedProteomicsReader(spectrumMetaData.GroupId, info.RawFile.Path);
// Then grab the actual spectrum...
var summary = new ScanSummary();
var spectrum = rawReader.GetRawSpectra(spectrumMetaData.Scan, 2, out summary);
// Then do what you want...
// Profit???
}
}
}
}
}
}
}
}
/// <summary>
/// Adds a Collection of Objects to the Database.
/// </summary>
/// <param name="tCollection">Collection of Objects to be added</param>
public new virtual void AddAllStateless(ICollection <UMCClusterLight> tCollection, IProgress <ProgressData> progress = null)
{
DatabaseIndexer.IndexClustersDrop(NHibernateUtil.Path);
base.AddAllStateless(tCollection, progress);
DatabaseIndexer.IndexClusters(NHibernateUtil.Path);
}
