/// <summary>
/// Adds and applies a single new clustering algorithm.
/// </summary>
public void AddClusterer(ConfigurationClusterer toAdd, ProgressReporter setProgress)
{
List <ConfigurationClusterer> existing = new List <ConfigurationClusterer>(this.Clusterers);
existing.Add(toAdd);
this.SetClusterers(existing.ToArray(), setProgress, false);
}
void create_clusters_from_pathways()
{
this.BeginWait("Create clusters from pathways", false);
var args = new ArgsClusterer(Algo.ID_PATFROMPATH, this._core.Matrices.First(), null, null, null, false, EClustererStatistics.None, null, "P:")
{
OverrideDisplayName = "create_clusters_from_pathways",
};
ConfigurationClusterer config = new ConfigurationClusterer()
{
Args = args
};
this._core.AddClusterer(config, new ProgressReporter(this));
this.EndWait();
}
/// <summary>IMPLEMENTS ClustererBase</summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix dmatrix, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
Dictionary <Cluster, Cluster> result = new Dictionary <Cluster, Cluster>();
WeakReference <ConfigurationClusterer> wrMethod = (WeakReference <ConfigurationClusterer>)args.Parameters[0];
ConfigurationClusterer existing = wrMethod.GetTargetOrThrow();
var existingResults = existing.Results;
foreach (Vector vector in vmatrix.Vectors)
{
Cluster existingCluster = existingResults.Assignments.FirstOrDefault(z => z.Peak == vector.Peak)?.Cluster;
if (existingCluster != null)
{
Cluster newCluster = result.GetOrCreate(existingCluster, xc => new Cluster(xc.ShortName, tag));
newCluster.Assignments.Add(new Assignment(vector, newCluster, double.NaN));
}
}
return(result.Values);
}
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix UNUSED, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
ConfigurationClusterer config = ((WeakReference <ConfigurationClusterer>)args.Parameters[0]).GetTargetOrThrow();
List <Cluster> myClusters = new List <Cluster>();
// Iterate existing clusters
prog.Enter("Iterating existing");
for (int index = 0; index < config.Results.Clusters.Length; index++)
{
Cluster cluster = config.Results.Clusters[index];
prog.SetProgress(index, config.Results.Clusters.Length);
if (!cluster.States.HasFlag(Session.Main.Cluster.EStates.Insignificants))
{
// Get the centre
IReadOnlyList <double> centre = cluster.GetCentre(ECentreMode.Average, ECandidateMode.Assignments)[0];
// Reorder the centre to match our vmatrix
// centre = Reorder(centre, config.Results.VMatrix.Conditions, vmatrix.Conditions);
Cluster myCluster = new Cluster(cluster.DisplayName, tag);
myCluster.Exemplars.Add(centre);
myClusters.Add(myCluster);
}
}
prog.Leave();
prog.Enter("Assigning peaks");
LegacyClustererHelper.Assign(vmatrix, myClusters, ECandidateMode.Exemplars, args.Distance, prog);
prog.Leave();
Cluster matchCluster = new Cluster("Matches", tag);
matchCluster.States |= Session.Main.Cluster.EStates.Insignificants;
return(myClusters);
}
/// <summary>IMPLEMENTS ClustererBase</summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix dmatrix, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
Cluster result = new Cluster("Com", tag);
foreach (Vector vector in vmatrix.Vectors)
{
result.Assignments.Add(new Assignment(vector, result, double.NaN));
}
return(new Cluster[] { result });
}
/// <summary>
/// ACTION!
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vm, DistanceMatrix dm, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
// Construct similarity matrix
int N = vm.NumRows;
double[,] s; // = new double[N, N];
double[,] r = new double[N, N];
double[,] a = new double[N, N];
double[,] rn = new double[N, N];
double[,] an = new double[N, N];
// CALCULATE SIMILARITIES "S"
s = dm.Values;
// CALCULATE PREFERENCES "diag(S)"
double median = ApMedian(s);
for (int i = 0; i < N; i++)
{
s[i, i] = median;
}
// SET LAMBDA (change rate)
const double lambda = 0.5;
prog.SetProgress(-1);
// CALCULATE NEXT R AND NEXT A
for (int iter = 0; iter < 100; iter++)
{
prog.Enter("Affinity Propagation Iteration " + iter);
// CALCULATE R
// r[i,k] = s[i,k]
// - max( a[i,kp] + s[i,kp] )
for (int i = 0; i < N; i++)
{
for (int k = 0; k < N; k++)
{
double v = double.MinValue;
for (int kp = 0; kp < N; kp++)
{
if (kp != k)
{
v = Math.Max(v, a[i, kp] + s[i, kp]);
}
}
rn[i, k] = s[i, k] - v;
}
}
for (int i = 0; i < N; i++)
{
for (int k = 0; k < N; k++)
{
r[i, k] = r[i, k] * lambda + rn[i, k] * (1 - lambda);
}
}
// CALCULATE A
// a[i, k] = min(0, r(k,k)
// + sum( max ( 0,
// r(ip, k)
// a[k, k] = sum( max( 0, r(ip, k ) )
for (int i = 0; i < N; i++)
{
for (int k = 0; k < N; k++)
{
if (i != k)
{
double v = 0;
for (int ip = 0; ip < N; ip++)
{
if (ip != i && ip != k)
{
v += Math.Max(0, r[ip, k]);
}
}
an[i, k] = Math.Min(0, r[k, k] + v);
}
else
{
double v = 0;
for (int ip = 0; ip < N; ip++)
{
if (ip != i && ip != k)
{
v += Math.Max(0, r[ip, k]);
}
}
an[k, k] = v;
}
}
}
for (int i = 0; i < N; i++)
{
for (int k = 0; k < N; k++)
{
a[i, k] = a[i, k] * lambda + an[i, k] * (1 - lambda);
}
}
prog.Leave();
}
// CALCULATE EXEMPLARS "E"
// the value of k that maximizes a(i,k) + r(i,k)
int[] exemplars = new int[N];
double[] scores = new double[N];
for (int i = 0; i < N; i++)
{
double maxVal = double.MinValue;
int maxK = -1;
for (int k = 0; k < N; k++)
{
double val = a[i, k] + r[i, k];
if (val > maxVal)
{
maxVal = val;
maxK = k;
}
}
exemplars[i] = maxK;
scores[i] = maxVal; // HIGHER is better
}
// CONVERT TO CLUSTERS
Dictionary <int, Cluster> dict = new Dictionary <int, Cluster>();
for (int pInd = 0; pInd < vm.NumRows; pInd++)
{
Vector vec = vm.Vectors[pInd];
int exe = exemplars[pInd];
Vector vecx = vm.Vectors[exe];
double score = scores[pInd]; // HIGHER is better
Cluster clu = dict.GetOrCreate(exe, x => new Cluster(vecx.Peak.DisplayName, tag));
clu.Assignments.Add(new Assignment(vec, clu, score));
}
return(dict.Values.OrderBy(z => z.DisplayName));
}
/// <summary>
///
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix dmatrix, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
ConfigurationClusterer existing = ((WeakReference <ConfigurationClusterer>)args.Parameters[0]).GetTarget();
List <List <Assignment> > uniqueCombinations = new List <List <Assignment> >();
List <Cluster> newClusters = new List <Cluster>();
List <ObservationInfo[]> observations = new List <ObservationInfo[]>();
var existingResults = existing.Results;
prog.Enter("Finding unique matches");
for (int row = 0; row < vmatrix.NumRows; row++)
{
Vector vector = vmatrix.Vectors[row];
Peak peak = vector.Peak;
prog.SetProgress(row, vmatrix.NumRows);
List <Assignment> assignments = new List <Assignment>(existingResults.Assignments
.Where(z => z.Peak == peak)
.OrderBy(z => z.Vector.Group.Order));
int index = FindMatch(uniqueCombinations, assignments);
Cluster pat;
if (index == -1)
{
uniqueCombinations.Add(assignments);
string name = StringHelper.ArrayToString <Assignment>(assignments, z => z.Vector.Group.DisplayShortName + "." + z.Cluster.ShortName, " / ");
pat = new Cluster(name, tag);
// Centre (merge centres)
IEnumerable <IReadOnlyList <double> > centres = assignments.Select(z => z.Cluster.Centres.First());
pat.Centres.Add(centres.SelectMany(z => z).ToArray());
// Vector (merge vectors)
if (assignments[0].Vector.Observations != null)
{
observations.Add(assignments.Select(z => z.Vector.Observations).SelectMany(z => z).ToArray());
}
else
{
observations.Add(null);
}
// Relations (all clusters)
pat.Related.AddRange(assignments.Select(z => z.Cluster).Unique());
foreach (Cluster pat2 in pat.Related)
{
if (!pat2.Related.Contains(pat))
{
pat2.Related.Add(pat);
}
}
index = newClusters.Count;
newClusters.Add(pat);
}
pat = newClusters[index];
double[] values = assignments.Select(z => z.Vector.Values).SelectMany(z => z).ToArray();
pat.Assignments.Add(new Assignment(vector, pat, assignments.Count));
}
prog.Leave();
return(newClusters);
}
/// <summary>
/// Actual test running
/// </summary>
private static ClusterEvaluationPointer RunTest(Core core, ClusterEvaluationPointer origPointer, ClusterEvaluationConfiguration test, ProgressReporter proggy, bool paranoid, int index, int of)
{
UiControls.Assert(core.FileNames.Session != null, "Didn't expect the session filename to be null for cluster evaluation.");
List <ClusterEvaluationParameterResult> results = new List <ClusterEvaluationParameterResult>();
// Iterate over parameters
for (int valueIndex = 0; valueIndex < test.ParameterValues.Length; valueIndex++)
{
object value = test.ParameterValues[valueIndex];
List <ResultClusterer> repetitions = new List <ResultClusterer>();
// Iterate over repetitions
for (int repetition = 0; repetition < test.NumberOfRepeats; repetition++)
{
proggy.Enter("Test " + index + "/" + of + ", parameter " + valueIndex + "/" + test.ParameterValues.Length + ", repetition " + repetition + "/" + test.NumberOfRepeats);
// Create config
string newName = AlgoParameterCollection.ParamToString(value) + " " + StringHelper.Circle(repetition + 1);
object[] copyOfParameters = test.ClustererConfiguration.Parameters.ToArray();
copyOfParameters[test.ParameterIndex] = value;
ArgsClusterer copyOfArgs = new ArgsClusterer(
test.ClustererConfiguration.Id,
test.ClustererConfiguration.SourceProvider,
test.ClustererConfiguration.PeakFilter,
test.ClustererConfiguration.Distance,
test.ClustererConfiguration.ObsFilter,
test.ClustererConfiguration.SplitGroups,
test.ClustererConfiguration.Statistics,
copyOfParameters,
test.ClustererConfiguration.OverrideShortName)
{
OverrideDisplayName = newName,
Comment = test.ClustererConfiguration.Comment
};
var copyOfConfig = new ConfigurationClusterer()
{
Args = copyOfArgs
};
// Try load previus result
ResultClusterer result = null;
if (paranoid)
{
result = TryLoadIntermediateResult(core, test.Guid, valueIndex, repetition, proggy);
}
if (result == null)
{
// DO CLUSTERING!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
proggy.Enter("Clustering");
result = copyOfConfig.Cluster(core, 0, proggy);
proggy.Leave();
if (paranoid)
{
SaveIntermediateResult(core, test.Guid, valueIndex, repetition, result, proggy);
}
}
// Add result
repetitions.Add(result);
string name = AlgoParameterCollection.ParamToString(value);
results.Add(new ClusterEvaluationParameterResult(name, test, valueIndex, repetitions));
proggy.Leave();
}
}
ClusterEvaluationResults final = new ClusterEvaluationResults(core, test, results);
string folder = UiControls.GetOrCreateFixedFolder(UiControls.EInitialFolder.Evaluations);
string sessionName = Path.GetFileNameWithoutExtension(core.FileNames.Session);
string fileName = core.Options.ClusteringEvaluationResultsFileName;
fileName = fileName.Replace("{SESSION}", sessionName);
fileName = fileName.Replace("{RESULTS}", folder + "\\");
fileName = UiControls.GetNewFile(fileName);
return(SaveResults(core, fileName, origPointer, final, proggy));
}
/// <summary>
///
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix UNUSED, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
// GET OPTIONS
int k = (int)args.Parameters[0];
// CREATE RANDOM CENTRES
Random rnd = new Random();
var potentialCentres = new List <Vector>(vmatrix.Vectors);
List <Cluster> clusters = new List <Cluster>();
for (int n = 0; n < k; n++)
{
int random = rnd.Next(potentialCentres.Count);
Cluster p = new Cluster((clusters.Count + 1).ToString(), tag);
Vector vec = potentialCentres[random];
p.Exemplars.Add(vec.Values);
potentialCentres.RemoveAt(random);
clusters.Add(p);
}
// Assign to exemplars
prog.Enter("Initialising assignments");
LegacyClustererHelper.Assign(vmatrix, clusters, ECandidateMode.Exemplars, args.Distance, prog);
prog.Leave();
// Centre
LegacyClustererHelper.PerformKMeansCentering(vmatrix, clusters, args.Distance, prog);
return(clusters);
}
/// <summary>
/// d-k-means++
/// Ignores insignificant variables.
/// Returns new clusters (these won't be added to the core so make sure to do so)
/// </summary>
private static List <Cluster> AutogenerateClusters(IntensityMatrix vmatrix, List <Cluster> seed, double?stoppingDistance, int?stoppingCount, ConfigurationMetric metric, ConfigurationClusterer tag, ProgressReporter prog)
{
// Make a log of whatever limits have been set
if (!stoppingCount.HasValue && !stoppingDistance.HasValue)
{
throw new InvalidOperationException("No stopping condition set.");
}
// Assign all variables to nearest
List <Cluster> result = new List <Cluster>(seed);
// Get the actual limits
int iterations = 0;
int count = (stoppingCount - seed.Count) ?? Int32.MaxValue;
double distance = stoppingDistance ?? Double.MinValue;
// Get the most distant variable
prog.Enter("Initialising assignments");
LegacyClustererHelper.Assign(vmatrix, result, ECandidateMode.Exemplars, metric, prog);
Assignment mostDistant = GetMostDistantAssignment(result);
prog.Leave();
// Continue until our limits are breached
while ((count > 0) && (mostDistant.Score > distance))
{
// Check we haven't got unreasonable limits
iterations++;
prog.Enter("Centre generation (iteration " + iterations + ")");
if (iterations > 1000)
{
throw new InvalidOperationException("Too many iterations - exiting.");
}
// Create a new cluster with the most distant variable as its exemplar
var newCluster = new Cluster((result.Count + 1).ToString(), tag);
result.Add(newCluster);
newCluster.Exemplars.Add(mostDistant.Vector.Values); // todo: check to prevent multiple assignment?
// Make the assignments based on the closest exemplars
LegacyClustererHelper.Assign(vmatrix, result, ECandidateMode.Exemplars, metric, prog);
// Basic check
if (!newCluster.Assignments.Vectors.Contains(mostDistant.Vector))
{
throw new InvalidOperationException("Problem creating new cluster from vector - " + mostDistant.Vector.ToString() + " doesn't like being in its own cluster. Check this vector for discrepancies.");
}
// Get the next most distant variable
count = count - 1;
mostDistant = GetMostDistantAssignment(result);
prog.Leave();
}
// Return the number of iterations
return(result);
}
/// <summary>
///
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix UNUSED, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
// Get parameters
// COUNT LIMIT
int countLimit = (int)tag.UntypedArgs.Parameters[0];
// DISTANCE LIMIT
double distanceLimit = (double)tag.UntypedArgs.Parameters[1];
// SEED PEAK
WeakReference <Peak> seedPeakRef = (WeakReference <Peak>)tag.UntypedArgs.Parameters[2];
Peak seedPeak = seedPeakRef.GetTargetOrThrow();
// SEED GROUP
GroupInfo groupInfo = (GroupInfo)tag.UntypedArgs.Parameters[3];
// DO-K-MEANS?
bool doKMeans = (bool)tag.UntypedArgs.Parameters[4];
// Create the seed cluster
Cluster seedCluster = new Cluster("1", tag);
List <Cluster> seedList = new List <Cluster> {
seedCluster
};
int seedIndex = vmatrix.FindIndex(new IntensityMatrix.RowHeader(seedPeak, args.SplitGroups ? groupInfo : null));
if (seedIndex == -1)
{
throw new InvalidOperationException($"The chosen peak {{{seedPeak}}} cannot be used a seed because it is not present in the value matrix. Please check that this peak has not been excluded by the filter condition {{{args.PeakFilter}}}.");
}
seedCluster.Exemplars.Add(vmatrix.Vectors[seedIndex]);
// Autogenerate the clusters
int? nCountLimit = (countLimit != Int32.MinValue) ? countLimit : (int?)null;
double?nDistanceLimit = (distanceLimit != Double.MinValue) ? countLimit : (double?)null;
List <Cluster> autoGenClusters = AutogenerateClusters(vmatrix, seedList, nDistanceLimit, nCountLimit, args.Distance, tag, prog);
// Do k-means (if requested)
if (doKMeans)
{
prog.Enter("k-means");
LegacyClustererHelper.PerformKMeansCentering(vmatrix, autoGenClusters, args.Distance, prog);
prog.Leave();
}
// Return full list
return(autoGenClusters);
}
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix dmatrix, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
object[] inputs =
{
_script.IsInputPresent(0) ? vmatrix.Values : null,
_script.IsInputPresent(1) ? dmatrix.Values : null
};
prog.Enter("Running script");
prog.SetProgressMarquee();
int[] clusters = Arr.Instance.RunScriptIntV(_script, inputs, args.Parameters).ToArray();
prog.Leave();
prog.Enter("Creating clusters");
var result = CreateClustersFromIntegers(vmatrix, clusters, tag);
prog.Leave();
return(result);
}
/// <summary>
/// Constructor.
/// </summary>
public Cluster(string shortName, ConfigurationClusterer creationReason)
{
this.ShortName = shortName;
this.Method = creationReason;
}
/// <summary>
///
/// </summary>
protected override IEnumerable <Cluster> Cluster(IntensityMatrix vmatrix, DistanceMatrix dmatrix, ArgsClusterer args, ConfigurationClusterer tag, ProgressReporter prog)
{
Dictionary <Pathway, Cluster> d = new Dictionary <Pathway, Cluster>();
List <Cluster> result = new List <Cluster>();
prog.Enter("Finding pathways");
for (int index = 0; index < vmatrix.NumRows; index++)
{
Vector vec = vmatrix.Vectors[index];
Peak peak = vec.Peak;
prog.SetProgress(index, vmatrix.NumRows);
foreach (Annotation c in peak.Annotations)
{
foreach (Pathway p in c.Compound.Pathways)
{
Cluster pat;
if (!d.TryGetValue(p, out pat))
{
pat = new Cluster(p.DefaultDisplayName, tag);
pat.States |= Session.Main.Cluster.EStates.Pathway;
result.Add(pat);
d.Add(p, pat);
}
if (!pat.Assignments.Peaks.Contains(peak))
{
pat.Assignments.Add(new Assignment(vec, pat, double.NaN));
}
}
}
}
prog.Leave();
return(result);
}
void wizard_OkClicked(object sender, EventArgs e)
{
// Check
if (!this._wizard.RevalidateAll())
{
FrmMsgBox.ShowError(this, "Not all options have been selected.");
return;
}
int param1_numClusters = this._radStopN.Checked ? int.Parse(this._txtStopN.Text) : int.MinValue;
double param2_distanceLimit = this._radStopD.Checked ? double.Parse(this._txtStopD.Text) : double.MinValue;
Debug.Assert(this._ecbSeedPeak.HasSelection, "Expected a seed peak to be selected");
WeakReference <Peak> param3_seedPeak = new WeakReference <Peak>(this._ecbSeedPeak.SelectedItem);
GroupInfo param4_seedGroup = NamedItem <GroupInfo> .Extract(this._lstGroups.SelectedItem);
int param5_doKMeans = this._radFinishK.Checked ? 1 : 0;
IMatrixProvider source = this._ecbSource.SelectedItem;
object[] parameters = { param1_numClusters, param2_distanceLimit, param3_seedPeak, param4_seedGroup, param5_doKMeans };
string name = "DK";
// Create a constraint that only allows overlapping timepoints
HashSet <int> overlappingPoints = new HashSet <int>();
var fil = this._ecbFilter.SelectedItem ?? ObsFilter.Empty;
var passed = fil.Test(source.Provide.Columns.Select(z => z.Observation)).Passed;
HashSet <GroupInfo> groups = new HashSet <GroupInfo>(passed.Select(z => z.Group));
bool needsExFilter = false;
foreach (int ctp in this._core.Times)
{
bool trueInAny = false;
bool falseInAny = false;
foreach (GroupInfo g in groups)
{
if (passed.Any(z => z.Group == g && z.Time == ctp))
{
trueInAny = true;
}
else
{
falseInAny = true;
}
}
if (trueInAny && !falseInAny) // i.e. true in all TT
{
overlappingPoints.Add(ctp);
}
else if (trueInAny) // i.e. true in one but not all TF
{
needsExFilter = true;
}
//else if (falseInAny) // False in all (accptable) FT
// else // No groups FF
}
ObsFilter trueFilter;
if (needsExFilter)
{
List <ObsFilter.Condition> conditions = new List <ObsFilter.Condition>
{
new ObsFilter.ConditionFilter(Filter.ELogicOperator.And, false,
this._ecbFilter.SelectedItem, true),
new ObsFilter.ConditionTime(Filter.ELogicOperator.And, false,
Filter.EElementOperator.Is,
overlappingPoints)
};
trueFilter = new ObsFilter(null, null, conditions);
this._core.AddObsFilter(trueFilter);
}
else
{
trueFilter = this._ecbFilter.SelectedItem;
}
ArgsClusterer args = new ArgsClusterer(
Algo.ID_DKMEANSPPWIZ,
this._ecbSource.SelectedItem,
this._ecbPeakFilter.SelectedItem,
new ConfigurationMetric()
{
Args = new ArgsMetric(this._ecbDistance.SelectedItem.Id, this._ecbSource.SelectedItem, this._ecbDistance.SelectedItem.Parameters.StringToParams(this._core, this._txtDistanceParams.Text))
},
trueFilter,
this._chkClusterIndividually.Checked,
EClustererStatistics.None,
parameters,
"DK")
{
OverrideDisplayName = name,
Comment = "Generated using wizard"
};
ConfigurationClusterer config = new ConfigurationClusterer()
{
Args = args
};
FrmWait.Show(this, "Clustering", null, z => this._core.AddClusterer(config, z));
this.DialogResult = DialogResult.OK;
}
