/// <summary>
/// Handles the Click event of the addExperimentButton control.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="EventArgs"/> instance containing the event data.</param>
private void addExperimentButton_Click(object sender, EventArgs e)
{
//
// Figure out wich supervised learning mode was selected
//
ContrastiveDivergenceLearningMode supervisedLearningMode = ContrastiveDivergenceLearningMode.FeedForward;
if (jointAlternativeRdioButton.Checked)
{
supervisedLearningMode = ContrastiveDivergenceLearningMode.JointAlternative;
}
else if (jointNormalRadioButton.Checked)
{
supervisedLearningMode = ContrastiveDivergenceLearningMode.JointNormal;
}
//
// Validation: ensure that joint-learning is only selected if there's a hidden layer
//
var networkParameters = this.createNetworkControl.GetNetworkParameters();
if (supervisedLearningMode != ContrastiveDivergenceLearningMode.FeedForward && networkParameters.HiddenLayerSizes.Length == 0)
{
MessageBox.Show("Joint learning requires at least one hidden layer");
return;
}
//
// Add the experiment to the list
//
this.experiments.Add(new Experiment()
{
NetworkParameters = networkParameters,
SupervisedLearningMode = supervisedLearningMode,
SupervisedEpochs = Convert.ToInt32(this.supervisedEpochsUpDown.Value),
UnsupervisedEpochs = Convert.ToInt32(this.unsupervisedEpochsUpDown.Value),
LearningProbability = Convert.ToDouble(this.learningProbabilityUpDown.Value)
});
//
// Resize the columns of the grid
//
foreach (var column in this.experimentDataGridView.Columns.Cast <DataGridViewColumn>())
{
column.AutoSizeMode = DataGridViewAutoSizeColumnMode.AllCells;
}
}
/// <summary>
/// Trains the network.
/// </summary>
/// <param name="samples">The samples to use for training.</param>
/// <param name="supervised">If set to <c>true</c> then supervised training is performed.</param>
/// <param name="progressObserver">A delegate to an observer function which tracks the training progress.</param>
/// <param name="supervisedLearningMode">The supervised learning mode.</param>
private void Train(Sample[] samples, bool supervised, Action <int> progressObserver, ContrastiveDivergenceLearningMode supervisedLearningMode)
{
bool joint = false;
if (supervised && (
supervisedLearningMode == ContrastiveDivergenceLearningMode.JointAlternative ||
supervisedLearningMode == ContrastiveDivergenceLearningMode.JointNormal))
{
joint = true;
}
//
// TODO: It might speed things up to re-use activation probability arrays instead of generating new ones all the time
// for the positive and negative phases.
//
double reconstructionError = 0;
double labelReconstructionError = 0;
//
// Convert the input data, which consists of byte-valued pixel intensities, into linear activation probabilities
//
var previousProbabilities = new double[samples.Length][];
for (int sampleIndex = 0; sampleIndex < samples.Length; sampleIndex++)
{
previousProbabilities[sampleIndex] = new double[samples[sampleIndex].SampleData.Length];
for (int unitIndex = 0; unitIndex < samples[sampleIndex].SampleData.Length; unitIndex++)
{
previousProbabilities[sampleIndex][unitIndex] = (double)samples[sampleIndex].SampleData[unitIndex] / byte.MaxValue;
}
}
List <int> labelUnits = null;
//
// Train each layer from left to right
//
for (int layer = 0; layer < this.LayerCount - 1; layer++)
{
//
// If we are doing unsupervised training, then we skip the last layer
// TODO: Train top layer as well during unsupervised learning?
//
if ((!supervised || joint) && layer == this.LayerCount - 2)
{
break;
}
//
// Set a couple of boolean variables which indicate whether or not supervised learning should take place in the current layer
//
bool supervisedLayer = supervised && layer == this.LayerCount - 2;
bool jointSupervisedLayer = supervised && joint && layer == this.LayerCount - 3;
//
// Construct an array that will hold the (positive) activation probabilities,
// which we will then use for training the subsequent layer
//
var currentProbabilities = new double[samples.Length][];
//
// Create the matrices that will hold the positive and negative phase data (for the learning signal)
//
var positiveProducts = new double[this.GetUnitCount(layer), this.GetUnitCount(layer + 1)];
var negativeProducts = new double[this.GetUnitCount(layer), this.GetUnitCount(layer + 1)];
double[,] jointPositiveProducts = null;
double[,] jointNegativeProducts = null;
if (jointSupervisedLayer)
{
jointPositiveProducts = new double[this.GetUnitCount(layer + 1), this.GetUnitCount(layer + 2)];
jointNegativeProducts = new double[this.GetUnitCount(layer + 1), this.GetUnitCount(layer + 2)];
}
//
// Go through every training sample
//
for (int sampleIndex = 0; sampleIndex < samples.Length; sampleIndex++)
{
//
// Get the positive activation probabilites from the previous layer
//
var positiveProbabilitiesLeft = previousProbabilities[sampleIndex];
//
// Propagate them to the right
//
double[] positiveProbabilitiesRight;
List <int> positiveActivationsRight;
if (supervisedLayer)
{
//
// Clamp the activations on the right to correspond to the units that represent the class of this training sample
// TODO: It would probably be more realistic to have a union of naturally activated units and those in the label group
//
positiveActivationsRight = this.GetLabelUnits(((LabelledSample)samples[sampleIndex]).Label);
positiveProbabilitiesRight = new double[this.GetUnitCount(layer + 1)];
foreach (var unit in positiveActivationsRight)
{
positiveProbabilitiesRight[unit] = 1.0;
}
}
else
{
if (jointSupervisedLayer)
{
labelUnits = this.GetLabelUnits(((LabelledSample)samples[sampleIndex]).Label);
}
//
// Perform several propagations (i.e. collect activation samples)
//
List <int>[] activationSamplesRight = new List <int> [this.probabilityEstimationSamples];
for (int i = 0; i < this.probabilityEstimationSamples; i++)
{
var positiveActivationsLeft = this.GetRandomActivations(positiveProbabilitiesLeft);
if (jointSupervisedLayer)
{
activationSamplesRight[i] = this.PropagateToMiddle(positiveActivationsLeft, labelUnits, layer);
}
else
{
activationSamplesRight[i] = this.PropagateRight(positiveActivationsLeft, layer);
}
}
//
// Estimate the activation probabilities and store them so that we can use them when training the subsequent layer
//
positiveProbabilitiesRight = currentProbabilities[sampleIndex] = this.EstimateProbabilities(
activationSamplesRight,
this.GetUnitCount(layer + 1));
//
// We need to pick a binary activation to use for the negative (reconstruction) phase
//
positiveActivationsRight = activationSamplesRight[0];
}
//
// If we are doing supervised learning, then we stop here since we only want to adjust the connections to the
// output layer
//
if (supervised && !supervisedLayer && !jointSupervisedLayer)
{
continue;
}
//
// Gather data for the negative phase
//
double[] negativeProbabilitiesLeft;
double[] negativeProbabilitiesRight;
double[] negativeProbabilitiesLabels = null;
if (jointSupervisedLayer && supervisedLearningMode == ContrastiveDivergenceLearningMode.JointNormal)
{
this.GetNegativePhaseProbabilities(positiveActivationsRight, layer, out negativeProbabilitiesLeft, out negativeProbabilitiesRight, out negativeProbabilitiesLabels);
}
else
{
this.GetNegativePhaseProbabilities(positiveActivationsRight, layer, Partition.Right, out negativeProbabilitiesLeft, out negativeProbabilitiesRight);
}
#if DEBUG
//
// Calculate and show the reconstruction error
//
reconstructionError += this.CalculateReconstructionError(positiveProbabilitiesLeft, negativeProbabilitiesLeft);
if ((sampleIndex + 1) % ErrorInterval == 0)
{
System.Diagnostics.Debug.WriteLine("Reconstruction error: " + (reconstructionError / ErrorInterval));
reconstructionError = 0;
}
#endif
//
// Multiply the positive, respectively negative, activation probability vectors to obtain matrices of the size
// |LEFT PARTITION| x |RIGHT PARTITION|
//
this.MultiplyVectors(positiveProbabilitiesLeft, positiveProbabilitiesRight, positiveProducts);
this.MultiplyVectors(negativeProbabilitiesLeft, negativeProbabilitiesRight, negativeProducts);
//
// Use the difference between these two matrices as a "learning signal"
//
this.UpdateEdges(positiveProducts.Subtract(negativeProducts, true), layer, joint);
//this.UpdateThresholdsCD(layer, positiveProbabilitiesLeft.Subtract(negativeProbabilitiesLeft));
//this.UpdateThresholdsCD(layer + 1, positiveProbabilitiesRight.Subtract(negativeProbabilitiesRight));
//
// Should we also adjust the edges going to the label units?
//
if (jointSupervisedLayer)
{
//
// Calculate the positive products for the last two layers
//
var positiveProbabilitiesLabels = new double[this.ClassCount * this.UnitsPerClass];
foreach (var labelUnit in labelUnits)
{
positiveProbabilitiesLabels[labelUnit] = 1.0;
}
this.MultiplyVectors(positiveProbabilitiesRight, positiveProbabilitiesLabels, jointPositiveProducts);
//
// Now get the negative products
//
if (supervisedLearningMode == ContrastiveDivergenceLearningMode.JointAlternative)
{
this.GetNegativePhaseProbabilities(positiveActivationsRight, layer + 1, Partition.Left, out negativeProbabilitiesLabels, out negativeProbabilitiesRight);
}
this.MultiplyVectors(negativeProbabilitiesRight, negativeProbabilitiesLabels, jointNegativeProducts);
//
// Use the difference between these two matrices as a "learning signal"
//
this.UpdateEdges(jointPositiveProducts.Subtract(jointNegativeProducts, true), layer + 1, joint);
//this.UpdateThresholdsCD(layer + 2, positiveProbabilitiesLabels.Subtract(negativeProbabilitiesLabels));
#if DEBUG
//
// Calculate and show the reconstruction error
//
labelReconstructionError += this.CalculateReconstructionError(positiveProbabilitiesLabels, negativeProbabilitiesLabels);
if ((sampleIndex + 1) % ErrorInterval == 0)
{
System.Diagnostics.Debug.WriteLine("Label reconstruction error: " + (labelReconstructionError / (ErrorInterval * this.UnitsPerClass)));
labelReconstructionError = 0;
}
#endif
}
if ((sampleIndex + 1) % 100 == 0)
{
//this.UpdateThresholdsEma1();
}
//
// Notify the progress observer function
//
if (progressObserver != null)
{
progressObserver(sampleIndex + 1);
}
}
//
// The 'output' activation probabilities of this layer become the 'input' probabilities for the subsequent layer
//
previousProbabilities = currentProbabilities;
}
}