/// <summary>
/// I wrote this when the program crashed after an hour.
/// Intermediate results can now be saved to avoid losing data.
/// </summary>
private static ResultClusterer TryLoadIntermediateResult(Core core, Guid guid, int value, int repetition, ProgressReporter proggy)
{
string fileName = UiControls.GetTemporaryFile("." + value + "." + repetition + ".intermediate.dat", guid);
if (!File.Exists(fileName))
{
return(null);
}
LookupByGuidSerialiser guidS = core.GetLookups();
proggy.Enter("Loading saved results");
ResultClusterer result;
try
{
result = XmlSettings.LoadOrDefault <ResultClusterer>(new FileDescriptor(fileName, SerialisationFormat.MSerialiserFastBinary), null, guidS, proggy);
}
catch
{
proggy.Leave(); // Result will probably be NULL rather than throwing an exception
return(null);
}
UiControls.Assert(!guidS.HasLookupTableChanged, "Didn't expect the GUID lookup table to change when loading data.");
proggy.Leave();
return(result);
}
/// <summary>
/// d-k-means++
/// Ignores insignificant variables.
/// Returns new clusters (these won't be added to the core so make sure to do so)
/// </summary>
private static List <Cluster> AutogenerateClusters(IntensityMatrix vmatrix, List <Cluster> seed, double?stoppingDistance, int?stoppingCount, ConfigurationMetric metric, ConfigurationClusterer tag, ProgressReporter prog)
{
// Make a log of whatever limits have been set
if (!stoppingCount.HasValue && !stoppingDistance.HasValue)
{
throw new InvalidOperationException("No stopping condition set.");
}
// Assign all variables to nearest
List <Cluster> result = new List <Cluster>(seed);
// Get the actual limits
int iterations = 0;
int count = (stoppingCount - seed.Count) ?? Int32.MaxValue;
double distance = stoppingDistance ?? Double.MinValue;
// Get the most distant variable
prog.Enter("Initialising assignments");
LegacyClustererHelper.Assign(vmatrix, result, ECandidateMode.Exemplars, metric, prog);
Assignment mostDistant = GetMostDistantAssignment(result);
prog.Leave();
// Continue until our limits are breached
while ((count > 0) && (mostDistant.Score > distance))
{
// Check we haven't got unreasonable limits
iterations++;
prog.Enter("Centre generation (iteration " + iterations + ")");
if (iterations > 1000)
{
throw new InvalidOperationException("Too many iterations - exiting.");
}
// Create a new cluster with the most distant variable as its exemplar
var newCluster = new Cluster((result.Count + 1).ToString(), tag);
result.Add(newCluster);
newCluster.Exemplars.Add(mostDistant.Vector.Values); // todo: check to prevent multiple assignment?
// Make the assignments based on the closest exemplars
LegacyClustererHelper.Assign(vmatrix, result, ECandidateMode.Exemplars, metric, prog);
// Basic check
if (!newCluster.Assignments.Vectors.Contains(mostDistant.Vector))
{
throw new InvalidOperationException("Problem creating new cluster from vector - " + mostDistant.Vector.ToString() + " doesn't like being in its own cluster. Check this vector for discrepancies.");
}
// Get the next most distant variable
count = count - 1;
mostDistant = GetMostDistantAssignment(result);
prog.Leave();
}
// Return the number of iterations
return(result);
}
/// <summary>
///
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix UNUSED, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
// GET OPTIONS
int k = (int)args.Parameters[0];
// CREATE RANDOM CENTRES
Random rnd = new Random();
var potentialCentres = new List <Vector>(vmatrix.Vectors);
List <Cluster> clusters = new List <Cluster>();
for (int n = 0; n < k; n++)
{
int random = rnd.Next(potentialCentres.Count);
Cluster p = new Cluster((clusters.Count + 1).ToString(), tag);
Vector vec = potentialCentres[random];
p.Exemplars.Add(vec.Values);
potentialCentres.RemoveAt(random);
clusters.Add(p);
}
// Assign to exemplars
prog.Enter("Initialising assignments");
LegacyClustererHelper.Assign(vmatrix, clusters, ECandidateMode.Exemplars, args.Distance, prog);
prog.Leave();
// Centre
LegacyClustererHelper.PerformKMeansCentering(vmatrix, clusters, args.Distance, prog);
return(clusters);
}
/// <summary>
/// K-means centering.
/// </summary>
public static void PerformKMeansCentering(IntensityMatrix vmatrix, IReadOnlyList <Cluster> toChoose, ConfigurationMetric metric, ProgressReporter prog)
{
int n = 0;
do
{
if (n != 0)
{
prog.Leave();
}
prog.Enter("k-means (iteration " + (++n) + ")");
} while (Assign(vmatrix, toChoose, ECandidateMode.Assignments, metric, prog));
prog.Leave();
}
/// <summary>
/// Returns the distance matrix for a set of peaks.
/// </summary>
public static DistanceMatrix Create(Core core, IntensityMatrix valueMatrix, ConfigurationMetric metric, ProgressReporter prog)
{
int n = valueMatrix.NumRows;
int bytesRequired = (n * n) * 8;
int mbRequired = bytesRequired / (1024 * 1024);
int limit = core.Options.ObjectSizeLimit * 1024 * 1024;
if (bytesRequired > limit)
{
// It ain't gonna happen. I'm not creating a distance matrix over 500Mb
throw new InvalidOperationException("n = " + n + " input vectors. n * n = " + (n * n) + " elements in the distance matrix. 8 bytes per element gives " + (n * n * 8) + " bytes, or " + mbRequired + " megabytes. ObjectSizeLimit is " + core.Options.ObjectSizeLimit + " Mb. Reduce the number of input vectors by filtering the data, or change the limit from the preferences menu. Some algorithms also have the option to disable the distance matrix.");
}
double[,] s = new double[n, n];
prog.Enter("Calculating distance matrix");
for (int i = 0; i < n; i++)
{
prog.SetProgress(i, n);
for (int j = 0; j < n; j++)
{
s[i, j] = metric.Calculate(valueMatrix.Vectors[i], valueMatrix.Vectors[j]);
}
}
prog.Leave();
return(new DistanceMatrix(s, valueMatrix));
}
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix dmatrix, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
object[] inputs =
{
_script.IsInputPresent(0) ? vmatrix.Values : null,
_script.IsInputPresent(1) ? dmatrix.Values : null
};
prog.Enter("Running script");
prog.SetProgressMarquee();
int[] clusters = Arr.Instance.RunScriptIntV(_script, inputs, args.Parameters).ToArray();
prog.Leave();
prog.Enter("Creating clusters");
var result = CreateClustersFromIntegers(vmatrix, clusters, tag);
prog.Leave();
return(result);
}
private void ExportSelected(ProgressReporter prog)
{
if (this._chkData.Checked)
{
prog.Enter("Intensities");
this.ExportData(this._txtData.Text, this._ecbIntensitySource.SelectedItem.Provide);
prog.Leave();
}
if (this._chkObs.Checked)
{
prog.Enter("Observations");
this.Export(this._txtObservations.Text, DataSet.ForObservations(this._core));
prog.Leave();
}
if (this._chkPeaks.Checked)
{
prog.Enter("Peaks");
this.Export(this._txtPeaks.Text, DataSet.ForPeaks(this._core));
prog.Leave();
}
if (this._chkClusters.Checked)
{
prog.Enter("Assignments");
this.ExportAssignments(this._txtClusters.Text);
prog.Leave();
}
if (this._chkOther.Checked)
{
foreach (var ooi in this._otherExports)
{
prog.Enter(ooi.DataSet.ToUiString());
this.Export(ooi);
prog.Leave();
}
}
}
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix UNUSED, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
ConfigurationClusterer config = ((WeakReference <ConfigurationClusterer>)args.Parameters[0]).GetTargetOrThrow();
List <Cluster> myClusters = new List <Cluster>();
// Iterate existing clusters
prog.Enter("Iterating existing");
for (int index = 0; index < config.Results.Clusters.Length; index++)
{
Cluster cluster = config.Results.Clusters[index];
prog.SetProgress(index, config.Results.Clusters.Length);
if (!cluster.States.HasFlag(Session.Main.Cluster.EStates.Insignificants))
{
// Get the centre
IReadOnlyList <double> centre = cluster.GetCentre(ECentreMode.Average, ECandidateMode.Assignments)[0];
// Reorder the centre to match our vmatrix
// centre = Reorder(centre, config.Results.VMatrix.Conditions, vmatrix.Conditions);
Cluster myCluster = new Cluster(cluster.DisplayName, tag);
myCluster.Exemplars.Add(centre);
myClusters.Add(myCluster);
}
}
prog.Leave();
prog.Enter("Assigning peaks");
LegacyClustererHelper.Assign(vmatrix, myClusters, ECandidateMode.Exemplars, args.Distance, prog);
prog.Leave();
Cluster matchCluster = new Cluster("Matches", tag);
matchCluster.States |= Session.Main.Cluster.EStates.Insignificants;
return(myClusters);
}
/// <summary>
/// I wrote this when the program crashed after an hour.
/// Intermediate results can now be saved to avoid losing data.
/// </summary>
private static void SaveIntermediateResult(Core core, Guid guid, int value, int repetition, ResultClusterer result, ProgressReporter proggy)
{
string fileName = UiControls.GetTemporaryFile("." + value + "." + repetition + ".intermediate.dat", guid);
LookupByGuidSerialiser guidS = core.GetLookups();
proggy.Enter("Saving intermediate results");
XmlSettings.Save <ResultClusterer>(new FileDescriptor(fileName, SerialisationFormat.MSerialiserFastBinary), result, guidS, proggy);
if (core.SetLookups(guidS))
{
// UIDs have changed
SaveSession(core, proggy);
}
proggy.Leave();
}
/// <summary>
///
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix UNUSED, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
// Get parameters
// COUNT LIMIT
int countLimit = (int)tag.UntypedArgs.Parameters[0];
// DISTANCE LIMIT
double distanceLimit = (double)tag.UntypedArgs.Parameters[1];
// SEED PEAK
WeakReference <Peak> seedPeakRef = (WeakReference <Peak>)tag.UntypedArgs.Parameters[2];
Peak seedPeak = seedPeakRef.GetTargetOrThrow();
// SEED GROUP
GroupInfo groupInfo = (GroupInfo)tag.UntypedArgs.Parameters[3];
// DO-K-MEANS?
bool doKMeans = (bool)tag.UntypedArgs.Parameters[4];
// Create the seed cluster
Cluster seedCluster = new Cluster("1", tag);
List <Cluster> seedList = new List <Cluster> {
seedCluster
};
int seedIndex = vmatrix.FindIndex(new IntensityMatrix.RowHeader(seedPeak, args.SplitGroups ? groupInfo : null));
if (seedIndex == -1)
{
throw new InvalidOperationException($"The chosen peak {{{seedPeak}}} cannot be used a seed because it is not present in the value matrix. Please check that this peak has not been excluded by the filter condition {{{args.PeakFilter}}}.");
}
seedCluster.Exemplars.Add(vmatrix.Vectors[seedIndex]);
// Autogenerate the clusters
int? nCountLimit = (countLimit != Int32.MinValue) ? countLimit : (int?)null;
double?nDistanceLimit = (distanceLimit != Double.MinValue) ? countLimit : (double?)null;
List <Cluster> autoGenClusters = AutogenerateClusters(vmatrix, seedList, nDistanceLimit, nCountLimit, args.Distance, tag, prog);
// Do k-means (if requested)
if (doKMeans)
{
prog.Enter("k-means");
LegacyClustererHelper.PerformKMeansCentering(vmatrix, autoGenClusters, args.Distance, prog);
prog.Leave();
}
// Return full list
return(autoGenClusters);
}
/// <summary>
/// Saves results to file
///
/// Returns pointer (unless originalPointer is NULL, in which case the action is assumed to be an export and is ignored).
/// </summary>
private static ClusterEvaluationPointer SaveResults(Core core, string fileName, ClusterEvaluationPointer originalPointer, ClusterEvaluationResults results, ProgressReporter proggy)
{
LookupByGuidSerialiser guidS = core.GetLookups();
proggy.Enter("Saving results");
XmlSettings.Save <ClusterEvaluationResults>(fileName, results, guidS, proggy);
if (core.SetLookups(guidS))
{
// UIDs have changed
SaveSession(core, proggy);
}
proggy.Leave();
if (originalPointer == null)
{
return(null);
}
return(new ClusterEvaluationPointer(fileName, originalPointer.Configuration));
}
/// <summary>
///
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix dmatrix, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
Dictionary <Pathway, Cluster> d = new Dictionary <Pathway, Cluster>();
List <Cluster> result = new List <Cluster>();
prog.Enter("Finding pathways");
for (int index = 0; index < vmatrix.NumRows; index++)
{
Vector vec = vmatrix.Vectors[index];
Peak peak = vec.Peak;
prog.SetProgress(index, vmatrix.NumRows);
foreach (Annotation c in peak.Annotations)
{
foreach (Pathway p in c.Compound.Pathways)
{
Cluster pat;
if (!d.TryGetValue(p, out pat))
{
pat = new Cluster(p.DefaultDisplayName, tag);
pat.States |= Session.Main.Cluster.EStates.Pathway;
result.Add(pat);
d.Add(p, pat);
}
if (!pat.Assignments.Peaks.Contains(peak))
{
pat.Assignments.Add(new Assignment(vec, pat, double.NaN));
}
}
}
}
prog.Leave();
return(result);
}
/// <summary>
/// ACTION!
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vm, DistanceMatrix dm, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
// Construct similarity matrix
int N = vm.NumRows;
double[,] s; // = new double[N, N];
double[,] r = new double[N, N];
double[,] a = new double[N, N];
double[,] rn = new double[N, N];
double[,] an = new double[N, N];
// CALCULATE SIMILARITIES "S"
s = dm.Values;
// CALCULATE PREFERENCES "diag(S)"
double median = ApMedian(s);
for (int i = 0; i < N; i++)
{
s[i, i] = median;
}
// SET LAMBDA (change rate)
const double lambda = 0.5;
prog.SetProgress(-1);
// CALCULATE NEXT R AND NEXT A
for (int iter = 0; iter < 100; iter++)
{
prog.Enter("Affinity Propagation Iteration " + iter);
// CALCULATE R
// r[i,k] = s[i,k]
// - max( a[i,kp] + s[i,kp] )
for (int i = 0; i < N; i++)
{
for (int k = 0; k < N; k++)
{
double v = double.MinValue;
for (int kp = 0; kp < N; kp++)
{
if (kp != k)
{
v = Math.Max(v, a[i, kp] + s[i, kp]);
}
}
rn[i, k] = s[i, k] - v;
}
}
for (int i = 0; i < N; i++)
{
for (int k = 0; k < N; k++)
{
r[i, k] = r[i, k] * lambda + rn[i, k] * (1 - lambda);
}
}
// CALCULATE A
// a[i, k] = min(0, r(k,k)
// + sum( max ( 0,
// r(ip, k)
// a[k, k] = sum( max( 0, r(ip, k ) )
for (int i = 0; i < N; i++)
{
for (int k = 0; k < N; k++)
{
if (i != k)
{
double v = 0;
for (int ip = 0; ip < N; ip++)
{
if (ip != i && ip != k)
{
v += Math.Max(0, r[ip, k]);
}
}
an[i, k] = Math.Min(0, r[k, k] + v);
}
else
{
double v = 0;
for (int ip = 0; ip < N; ip++)
{
if (ip != i && ip != k)
{
v += Math.Max(0, r[ip, k]);
}
}
an[k, k] = v;
}
}
}
for (int i = 0; i < N; i++)
{
for (int k = 0; k < N; k++)
{
a[i, k] = a[i, k] * lambda + an[i, k] * (1 - lambda);
}
}
prog.Leave();
}
// CALCULATE EXEMPLARS "E"
// the value of k that maximizes a(i,k) + r(i,k)
int[] exemplars = new int[N];
double[] scores = new double[N];
for (int i = 0; i < N; i++)
{
double maxVal = double.MinValue;
int maxK = -1;
for (int k = 0; k < N; k++)
{
double val = a[i, k] + r[i, k];
if (val > maxVal)
{
maxVal = val;
maxK = k;
}
}
exemplars[i] = maxK;
scores[i] = maxVal; // HIGHER is better
}
// CONVERT TO CLUSTERS
Dictionary <int, Cluster> dict = new Dictionary <int, Cluster>();
for (int pInd = 0; pInd < vm.NumRows; pInd++)
{
Vector vec = vm.Vectors[pInd];
int exe = exemplars[pInd];
Vector vecx = vm.Vectors[exe];
double score = scores[pInd]; // HIGHER is better
Cluster clu = dict.GetOrCreate(exe, x => new Cluster(vecx.Peak.DisplayName, tag));
clu.Assignments.Add(new Assignment(vec, clu, score));
}
return(dict.Values.OrderBy(z => z.DisplayName));
}
/// <summary>
///
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix dmatrix, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
ConfigurationClusterer existing = ((WeakReference <ConfigurationClusterer>)args.Parameters[0]).GetTarget();
List <List <Assignment> > uniqueCombinations = new List <List <Assignment> >();
List <Cluster> newClusters = new List <Cluster>();
List <ObservationInfo[]> observations = new List <ObservationInfo[]>();
var existingResults = existing.Results;
prog.Enter("Finding unique matches");
for (int row = 0; row < vmatrix.NumRows; row++)
{
Vector vector = vmatrix.Vectors[row];
Peak peak = vector.Peak;
prog.SetProgress(row, vmatrix.NumRows);
List <Assignment> assignments = new List <Assignment>(existingResults.Assignments
.Where(z => z.Peak == peak)
.OrderBy(z => z.Vector.Group.Order));
int index = FindMatch(uniqueCombinations, assignments);
Cluster pat;
if (index == -1)
{
uniqueCombinations.Add(assignments);
string name = StringHelper.ArrayToString <Assignment>(assignments, z => z.Vector.Group.DisplayShortName + "." + z.Cluster.ShortName, " / ");
pat = new Cluster(name, tag);
// Centre (merge centres)
IEnumerable <IReadOnlyList <double> > centres = assignments.Select(z => z.Cluster.Centres.First());
pat.Centres.Add(centres.SelectMany(z => z).ToArray());
// Vector (merge vectors)
if (assignments[0].Vector.Observations != null)
{
observations.Add(assignments.Select(z => z.Vector.Observations).SelectMany(z => z).ToArray());
}
else
{
observations.Add(null);
}
// Relations (all clusters)
pat.Related.AddRange(assignments.Select(z => z.Cluster).Unique());
foreach (Cluster pat2 in pat.Related)
{
if (!pat2.Related.Contains(pat))
{
pat2.Related.Add(pat);
}
}
index = newClusters.Count;
newClusters.Add(pat);
}
pat = newClusters[index];
double[] values = assignments.Select(z => z.Vector.Values).SelectMany(z => z).ToArray();
pat.Assignments.Add(new Assignment(vector, pat, assignments.Count));
}
prog.Leave();
return(newClusters);
}
private string BatchProcess(EUpdateResults options, IEnumerable <ClusterEvaluationPointer> tests, EClustererStatistics stats, ProgressReporter proggy)
{
StringBuilder sb = new StringBuilder();
foreach (ClusterEvaluationPointer res in tests)
{
sb.AppendLine("CONFIG: " + res.Configuration.ParameterConfigAsString);
sb.AppendLine("PARAMS: " + res.Configuration.ParameterValuesAsString);
sb.AppendLine("NAME: " + res.OverrideDisplayName);
sb.AppendLine("FILE: " + res.FileName);
if (!res.HasResults)
{
sb.AppendLine(" - No results.");
continue;
}
Stopwatch timer = Stopwatch.StartNew();
proggy.Enter("Loading results");
bool load = options.Has(EUpdateResults.Csv | EUpdateResults.Statistics | EUpdateResults.Resave);
ClusterEvaluationResults set = load ? this.LoadResults(res.FileName, proggy) : null;
proggy.Leave();
if (load && set == null)
{
sb.AppendLine(" - Load failed.");
continue;
}
sb.AppendLine(" - LOAD-TIME: " + timer.Elapsed);
sb.AppendLine();
if (options.Has(EUpdateResults.Csv))
{
timer.Restart();
proggy.Enter("Selecting results");
proggy.SetProgressMarquee();
this.Invoke((MethodInvoker)(() => this.SelectResults(res.FileName, set)));
proggy.Leave();
sb.AppendLine(" - DISPLAY-TIME: " + timer.Elapsed);
string csvFileName = Path.Combine(Path.GetDirectoryName(res.FileName), Path.GetFileNameWithoutExtension(res.FileName) + ".csv");
csvFileName = UiControls.GetNewFile(csvFileName, checkOriginal: true);
try
{
timer.Restart();
proggy.Enter("Exporting CSV");
proggy.SetProgressMarquee();
this.Invoke((MethodInvoker)(() =>
{
using (StreamWriter sw = new StreamWriter(csvFileName))
{
this._lvhStatistics.WriteItems(sw, true);
}
}));
proggy.Leave();
sb.AppendLine(" - EXPORT-TIME: " + timer.Elapsed);
sb.AppendLine(" - EXPORT: " + csvFileName);
}
catch (Exception ex)
{
sb.AppendLine(" - Export failed: " + ex.Message);
}
sb.AppendLine();
}
if (options.Has(EUpdateResults.Statistics))
{
foreach (ClusterEvaluationParameterResult rep in set.Results)
{
rep.RecalculateStatistics(this._core, stats, proggy);
}
}
if (options.Has(EUpdateResults.Information))
{
proggy.Enter("Exporting information");
string infoFileName = Path.Combine(Path.GetDirectoryName(res.FileName), Path.GetFileNameWithoutExtension(res.FileName) + ".txt");
infoFileName = UiControls.GetNewFile(infoFileName, checkOriginal: true);
StringBuilder info = new StringBuilder();
info.Append(res.DisplayName);
info.AppendLine(res.Configuration.ParameterConfigAsString);
info.AppendLine(res.Configuration.ParameterValuesAsString);
File.WriteAllText(infoFileName, info.ToString());
sb.AppendLine(" - INFORMATION: " + infoFileName);
sb.AppendLine();
}
if (options.Has(EUpdateResults.Resave))
{
string bakFileName = Path.Combine(Path.GetDirectoryName(res.FileName), Path.GetFileNameWithoutExtension(res.FileName) + ".old");
bakFileName = UiControls.GetNewFile(bakFileName, checkOriginal: true);
timer.Restart();
proggy.Enter("Backing up original");
proggy.SetProgressMarquee();
File.Copy(res.FileName, bakFileName, false);
proggy.Leave();
sb.AppendLine(" - BACKUP-TIME: " + timer.Elapsed);
sb.AppendLine(" - BACKUP: " + bakFileName);
timer.Restart();
proggy.Enter("Saving in latest format");
SaveResults(this._core, res.FileName, null, set, proggy);
proggy.Leave();
sb.AppendLine(" - SAVE-TIME: " + timer.Elapsed);
sb.AppendLine(" - SAVE: " + res.FileName);
sb.AppendLine();
}
}
return(sb.ToString());
}
/// <summary>
/// Recalculates the statistics.
/// </summary>
/// <param name="core">Core</param>
/// <param name="metric">Metric for statistics</param>
/// <param name="statistics">What to calculate</param>
/// <param name="prog">Report progress to</param>
/// <param name="vmatrix">Value matrix</param>
/// <param name="dmatrix">Distance matrix (optional - if not present will be calculated if necessary)</param>
internal void RecalculateStatistics(Core core, ConfigurationMetric metric, IntensityMatrix vmatrix, DistanceMatrix dmatrix, EClustererStatistics statistics, ProgressReporter prog)
{
// Add basics
ClustererStatistics[STAT_NUM_VECTORS] = vmatrix.NumRows;
ClustererStatistics[STAT_LENGTH_OF_VECTORS] = vmatrix.NumCols;
// Don't calculate metrics?
if (statistics == EClustererStatistics.None)
{
return;
}
// Get the non-insig clusters
Cluster[] realClusters = RealClusters.ToArray();
// If we don't have a DMatrix we should calculate the sil. width manually
// The DMatrix might be too big to pass to R so its better just to avoid it.
prog.Enter("Calculating statistics");
List <ObsFilter> groupFilters = new List <ObsFilter>();
// No filter
groupFilters.Add(null);
if (!vmatrix.HasSplitGroups)
{
// Defined filters
if (statistics.HasFlag(EClustererStatistics.IncludePartialVectorsForFilters))
{
groupFilters.AddRange(core.ObsFilters);
}
// Group filters (if not already)
if (statistics.HasFlag(EClustererStatistics.IncludePartialVectorsForGroups))
{
AllGroupsFilters(core, groupFilters);
}
}
List <ForStat> needsCalculating = new List <ForStat>();
prog.Enter("Input vectors");
ProgressParallelHandler progP = prog.CreateParallelHandler(groupFilters.Count);
ProgressParallelHandler closure1 = progP;
Parallel.ForEach(groupFilters, obsFilter => Thread_AddFilterToCalculationList(core, metric, vmatrix, dmatrix, statistics, realClusters, obsFilter, needsCalculating, closure1));
prog.Leave();
// ASSIGNMENT STATS
prog.Enter("Assignments");
progP = prog.CreateParallelHandler(needsCalculating.Count);
ProgressParallelHandler closure2 = progP;
Parallel.ForEach(needsCalculating, z => Thread_CalculateAssignmentStatistics(statistics, z, realClusters, metric, closure2));
prog.Leave();
// CLUSTER STATS
prog.Enter("Clusters");
progP = prog.CreateParallelHandler(this.Clusters.Length);
Parallel.ForEach(this.Clusters, z => Thread_CalculateClusterStatistics(core, statistics, z, progP));
prog.Leave();
// SUMMARY STATS
prog.Enter("Summary");
CalculateSummaryStatistics(core, statistics, realClusters);
prog.Leave();
prog.Leave();
}
/// <summary>
/// Saves the core after GUIDs have changed.
/// </summary>
private static void SaveSession(Core core, ProgressReporter proggy)
{
proggy.Enter("Saving session");
core.Save(core.FileNames.Session, proggy);
proggy.Leave();
}
/// <summary>
/// Actual test running
/// </summary>
private static ClusterEvaluationPointer RunTest(Core core, ClusterEvaluationPointer origPointer, ClusterEvaluationConfiguration test, ProgressReporter proggy, bool paranoid, int index, int of)
{
UiControls.Assert(core.FileNames.Session != null, "Didn't expect the session filename to be null for cluster evaluation.");
List <ClusterEvaluationParameterResult> results = new List <ClusterEvaluationParameterResult>();
// Iterate over parameters
for (int valueIndex = 0; valueIndex < test.ParameterValues.Length; valueIndex++)
{
object value = test.ParameterValues[valueIndex];
List <ResultClusterer> repetitions = new List <ResultClusterer>();
// Iterate over repetitions
for (int repetition = 0; repetition < test.NumberOfRepeats; repetition++)
{
proggy.Enter("Test " + index + "/" + of + ", parameter " + valueIndex + "/" + test.ParameterValues.Length + ", repetition " + repetition + "/" + test.NumberOfRepeats);
// Create config
string newName = AlgoParameterCollection.ParamToString(value) + " " + StringHelper.Circle(repetition + 1);
object[] copyOfParameters = test.ClustererConfiguration.Parameters.ToArray();
copyOfParameters[test.ParameterIndex] = value;
ArgsClusterer copyOfArgs = new ArgsClusterer(
test.ClustererConfiguration.Id,
test.ClustererConfiguration.SourceProvider,
test.ClustererConfiguration.PeakFilter,
test.ClustererConfiguration.Distance,
test.ClustererConfiguration.ObsFilter,
test.ClustererConfiguration.SplitGroups,
test.ClustererConfiguration.Statistics,
copyOfParameters,
test.ClustererConfiguration.OverrideShortName)
{
OverrideDisplayName = newName,
Comment = test.ClustererConfiguration.Comment
};
var copyOfConfig = new ConfigurationClusterer()
{
Args = copyOfArgs
};
// Try load previus result
ResultClusterer result = null;
if (paranoid)
{
result = TryLoadIntermediateResult(core, test.Guid, valueIndex, repetition, proggy);
}
if (result == null)
{
// DO CLUSTERING!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
proggy.Enter("Clustering");
result = copyOfConfig.Cluster(core, 0, proggy);
proggy.Leave();
if (paranoid)
{
SaveIntermediateResult(core, test.Guid, valueIndex, repetition, result, proggy);
}
}
// Add result
repetitions.Add(result);
string name = AlgoParameterCollection.ParamToString(value);
results.Add(new ClusterEvaluationParameterResult(name, test, valueIndex, repetitions));
proggy.Leave();
}
}
ClusterEvaluationResults final = new ClusterEvaluationResults(core, test, results);
string folder = UiControls.GetOrCreateFixedFolder(UiControls.EInitialFolder.Evaluations);
string sessionName = Path.GetFileNameWithoutExtension(core.FileNames.Session);
string fileName = core.Options.ClusteringEvaluationResultsFileName;
fileName = fileName.Replace("{SESSION}", sessionName);
fileName = fileName.Replace("{RESULTS}", folder + "\\");
fileName = UiControls.GetNewFile(fileName);
return(SaveResults(core, fileName, origPointer, final, proggy));
}
public ResultClusterer ExecuteAlgorithm(Core core, int isPreview, bool doNotCluster, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog, out IntensityMatrix vmatrixOut, out DistanceMatrix dmatrixOut)
{
IReadOnlyList <Peak> peaks;
if (isPreview > 0 && isPreview < core.Peaks.Count)
{
List <Peak> p = core.Peaks.ToList();
p.Shuffle();
p = p.GetRange(0, Math.Min(isPreview, p.Count)).ToList();
// Make sure any seed peaks are in the list
foreach (Peak peak in tag.Args.Parameters.OfType <WeakReference <Peak> >().Select(par => (par).GetTargetOrThrow()))
{
p.Insert(0, peak);
p.RemoveAt(p.Count - 1);
}
peaks = p;
}
else
{
peaks = core.Peaks;
}
// FILTER PEAKS
PeakFilter pfilter = args.PeakFilter ?? PeakFilter.Empty;
IntensityMatrix src = args.SourceMatrix;
Filter <Peak> .Results filter = pfilter.Test(peaks);
Cluster insigs;
if (filter.Failed.Count == 0)
{
insigs = null;
}
else
{
insigs = new Cluster("Insig", tag);
insigs.States |= Session.Main.Cluster.EStates.Insignificants;
// We still need the vmatrix for plotting later
IntensityMatrix operational = src.Subset(args.PeakFilter, args.ObsFilter, ESubsetFlags.InvertPeakFilter);
if (args.SplitGroups)
{
operational = operational.SplitGroups();
}
for (int index = 0; index < operational.NumRows; index++)
{
Vector p = new Vector(operational, index);
insigs.Assignments.Add(new Assignment(p, insigs, double.NaN));
}
}
// CREATE VMATRIX AND FILTER OBSERVATIONS
PeakFilter temp = new PeakFilter("filtered in", null, new[] { new PeakFilter.ConditionPeak(Filter.ELogicOperator.And, false, filter.Failed, Filter.EElementOperator.IsNot) });
IntensityMatrix vmatrix = src.Subset(args.PeakFilter, args.ObsFilter, ESubsetFlags.None);
if (args.SplitGroups)
{
vmatrix = vmatrix.SplitGroups();
}
prog.Enter("Creating distance matrix");
DistanceMatrix dmatrix = RequiresDistanceMatrix ? DistanceMatrix.Create(core, vmatrix, args.Distance, prog) : null;
prog.Leave();
IEnumerable <Cluster> clusters;
if (doNotCluster)
{
vmatrixOut = vmatrix;
dmatrixOut = dmatrix;
return(null);
}
// CLUSTER USING VMATRIX OR DMATRIX
prog.Enter("Clustering");
clusters = Cluster(vmatrix, dmatrix, args, tag, prog);
prog.Leave();
vmatrixOut = vmatrix;
dmatrixOut = dmatrix;
List <Cluster> result = new List <Cluster>();
if (insigs != null)
{
result.Add(insigs);
}
result.AddRange(clusters);
return(new ResultClusterer(result));
}
