/// <summary> /// Handles the Click event of the addExperimentButton control. /// </summary> /// <param name="sender">The source of the event.</param> /// <param name="e">The <see cref="EventArgs"/> instance containing the event data.</param> private void addExperimentButton_Click(object sender, EventArgs e) { // // Figure out wich supervised learning mode was selected // ContrastiveDivergenceLearningMode supervisedLearningMode = ContrastiveDivergenceLearningMode.FeedForward; if (jointAlternativeRdioButton.Checked) { supervisedLearningMode = ContrastiveDivergenceLearningMode.JointAlternative; } else if (jointNormalRadioButton.Checked) { supervisedLearningMode = ContrastiveDivergenceLearningMode.JointNormal; } // // Validation: ensure that joint-learning is only selected if there's a hidden layer // var networkParameters = this.createNetworkControl.GetNetworkParameters(); if (supervisedLearningMode != ContrastiveDivergenceLearningMode.FeedForward && networkParameters.HiddenLayerSizes.Length == 0) { MessageBox.Show("Joint learning requires at least one hidden layer"); return; } // // Add the experiment to the list // this.experiments.Add(new Experiment() { NetworkParameters = networkParameters, SupervisedLearningMode = supervisedLearningMode, SupervisedEpochs = Convert.ToInt32(this.supervisedEpochsUpDown.Value), UnsupervisedEpochs = Convert.ToInt32(this.unsupervisedEpochsUpDown.Value), LearningProbability = Convert.ToDouble(this.learningProbabilityUpDown.Value) }); // // Resize the columns of the grid // foreach (var column in this.experimentDataGridView.Columns.Cast <DataGridViewColumn>()) { column.AutoSizeMode = DataGridViewAutoSizeColumnMode.AllCells; } }
/// <summary> /// Trains the network. /// </summary> /// <param name="samples">The samples to use for training.</param> /// <param name="supervised">If set to <c>true</c> then supervised training is performed.</param> /// <param name="progressObserver">A delegate to an observer function which tracks the training progress.</param> /// <param name="supervisedLearningMode">The supervised learning mode.</param> private void Train(Sample[] samples, bool supervised, Action <int> progressObserver, ContrastiveDivergenceLearningMode supervisedLearningMode) { bool joint = false; if (supervised && ( supervisedLearningMode == ContrastiveDivergenceLearningMode.JointAlternative || supervisedLearningMode == ContrastiveDivergenceLearningMode.JointNormal)) { joint = true; } // // TODO: It might speed things up to re-use activation probability arrays instead of generating new ones all the time // for the positive and negative phases. // double reconstructionError = 0; double labelReconstructionError = 0; // // Convert the input data, which consists of byte-valued pixel intensities, into linear activation probabilities // var previousProbabilities = new double[samples.Length][]; for (int sampleIndex = 0; sampleIndex < samples.Length; sampleIndex++) { previousProbabilities[sampleIndex] = new double[samples[sampleIndex].SampleData.Length]; for (int unitIndex = 0; unitIndex < samples[sampleIndex].SampleData.Length; unitIndex++) { previousProbabilities[sampleIndex][unitIndex] = (double)samples[sampleIndex].SampleData[unitIndex] / byte.MaxValue; } } List <int> labelUnits = null; // // Train each layer from left to right // for (int layer = 0; layer < this.LayerCount - 1; layer++) { // // If we are doing unsupervised training, then we skip the last layer // TODO: Train top layer as well during unsupervised learning? // if ((!supervised || joint) && layer == this.LayerCount - 2) { break; } // // Set a couple of boolean variables which indicate whether or not supervised learning should take place in the current layer // bool supervisedLayer = supervised && layer == this.LayerCount - 2; bool jointSupervisedLayer = supervised && joint && layer == this.LayerCount - 3; // // Construct an array that will hold the (positive) activation probabilities, // which we will then use for training the subsequent layer // var currentProbabilities = new double[samples.Length][]; // // Create the matrices that will hold the positive and negative phase data (for the learning signal) // var positiveProducts = new double[this.GetUnitCount(layer), this.GetUnitCount(layer + 1)]; var negativeProducts = new double[this.GetUnitCount(layer), this.GetUnitCount(layer + 1)]; double[,] jointPositiveProducts = null; double[,] jointNegativeProducts = null; if (jointSupervisedLayer) { jointPositiveProducts = new double[this.GetUnitCount(layer + 1), this.GetUnitCount(layer + 2)]; jointNegativeProducts = new double[this.GetUnitCount(layer + 1), this.GetUnitCount(layer + 2)]; } // // Go through every training sample // for (int sampleIndex = 0; sampleIndex < samples.Length; sampleIndex++) { // // Get the positive activation probabilites from the previous layer // var positiveProbabilitiesLeft = previousProbabilities[sampleIndex]; // // Propagate them to the right // double[] positiveProbabilitiesRight; List <int> positiveActivationsRight; if (supervisedLayer) { // // Clamp the activations on the right to correspond to the units that represent the class of this training sample // TODO: It would probably be more realistic to have a union of naturally activated units and those in the label group // positiveActivationsRight = this.GetLabelUnits(((LabelledSample)samples[sampleIndex]).Label); positiveProbabilitiesRight = new double[this.GetUnitCount(layer + 1)]; foreach (var unit in positiveActivationsRight) { positiveProbabilitiesRight[unit] = 1.0; } } else { if (jointSupervisedLayer) { labelUnits = this.GetLabelUnits(((LabelledSample)samples[sampleIndex]).Label); } // // Perform several propagations (i.e. collect activation samples) // List <int>[] activationSamplesRight = new List <int> [this.probabilityEstimationSamples]; for (int i = 0; i < this.probabilityEstimationSamples; i++) { var positiveActivationsLeft = this.GetRandomActivations(positiveProbabilitiesLeft); if (jointSupervisedLayer) { activationSamplesRight[i] = this.PropagateToMiddle(positiveActivationsLeft, labelUnits, layer); } else { activationSamplesRight[i] = this.PropagateRight(positiveActivationsLeft, layer); } } // // Estimate the activation probabilities and store them so that we can use them when training the subsequent layer // positiveProbabilitiesRight = currentProbabilities[sampleIndex] = this.EstimateProbabilities( activationSamplesRight, this.GetUnitCount(layer + 1)); // // We need to pick a binary activation to use for the negative (reconstruction) phase // positiveActivationsRight = activationSamplesRight[0]; } // // If we are doing supervised learning, then we stop here since we only want to adjust the connections to the // output layer // if (supervised && !supervisedLayer && !jointSupervisedLayer) { continue; } // // Gather data for the negative phase // double[] negativeProbabilitiesLeft; double[] negativeProbabilitiesRight; double[] negativeProbabilitiesLabels = null; if (jointSupervisedLayer && supervisedLearningMode == ContrastiveDivergenceLearningMode.JointNormal) { this.GetNegativePhaseProbabilities(positiveActivationsRight, layer, out negativeProbabilitiesLeft, out negativeProbabilitiesRight, out negativeProbabilitiesLabels); } else { this.GetNegativePhaseProbabilities(positiveActivationsRight, layer, Partition.Right, out negativeProbabilitiesLeft, out negativeProbabilitiesRight); } #if DEBUG // // Calculate and show the reconstruction error // reconstructionError += this.CalculateReconstructionError(positiveProbabilitiesLeft, negativeProbabilitiesLeft); if ((sampleIndex + 1) % ErrorInterval == 0) { System.Diagnostics.Debug.WriteLine("Reconstruction error: " + (reconstructionError / ErrorInterval)); reconstructionError = 0; } #endif // // Multiply the positive, respectively negative, activation probability vectors to obtain matrices of the size // |LEFT PARTITION| x |RIGHT PARTITION| // this.MultiplyVectors(positiveProbabilitiesLeft, positiveProbabilitiesRight, positiveProducts); this.MultiplyVectors(negativeProbabilitiesLeft, negativeProbabilitiesRight, negativeProducts); // // Use the difference between these two matrices as a "learning signal" // this.UpdateEdges(positiveProducts.Subtract(negativeProducts, true), layer, joint); //this.UpdateThresholdsCD(layer, positiveProbabilitiesLeft.Subtract(negativeProbabilitiesLeft)); //this.UpdateThresholdsCD(layer + 1, positiveProbabilitiesRight.Subtract(negativeProbabilitiesRight)); // // Should we also adjust the edges going to the label units? // if (jointSupervisedLayer) { // // Calculate the positive products for the last two layers // var positiveProbabilitiesLabels = new double[this.ClassCount * this.UnitsPerClass]; foreach (var labelUnit in labelUnits) { positiveProbabilitiesLabels[labelUnit] = 1.0; } this.MultiplyVectors(positiveProbabilitiesRight, positiveProbabilitiesLabels, jointPositiveProducts); // // Now get the negative products // if (supervisedLearningMode == ContrastiveDivergenceLearningMode.JointAlternative) { this.GetNegativePhaseProbabilities(positiveActivationsRight, layer + 1, Partition.Left, out negativeProbabilitiesLabels, out negativeProbabilitiesRight); } this.MultiplyVectors(negativeProbabilitiesRight, negativeProbabilitiesLabels, jointNegativeProducts); // // Use the difference between these two matrices as a "learning signal" // this.UpdateEdges(jointPositiveProducts.Subtract(jointNegativeProducts, true), layer + 1, joint); //this.UpdateThresholdsCD(layer + 2, positiveProbabilitiesLabels.Subtract(negativeProbabilitiesLabels)); #if DEBUG // // Calculate and show the reconstruction error // labelReconstructionError += this.CalculateReconstructionError(positiveProbabilitiesLabels, negativeProbabilitiesLabels); if ((sampleIndex + 1) % ErrorInterval == 0) { System.Diagnostics.Debug.WriteLine("Label reconstruction error: " + (labelReconstructionError / (ErrorInterval * this.UnitsPerClass))); labelReconstructionError = 0; } #endif } if ((sampleIndex + 1) % 100 == 0) { //this.UpdateThresholdsEma1(); } // // Notify the progress observer function // if (progressObserver != null) { progressObserver(sampleIndex + 1); } } // // The 'output' activation probabilities of this layer become the 'input' probabilities for the subsequent layer // previousProbabilities = currentProbabilities; } }