private void btn_forwardProp_Click(object sender, EventArgs e)
{
if (!chk_autoForwardProp.Checked)
{
return;
}
OneToManyMap <GraphMap <FeatureVector, WeightMatrix> .ILinkable, Pair <GraphMap <FeatureVector, WeightMatrix> .ILinkable, GraphMap <FeatureVector, WeightMatrix> .Link <WeightMatrix> > > backwards;
Queue <GraphMap <FeatureVector, WeightMatrix> .ILinkable> _;
Queue <GraphMap <FeatureVector, WeightMatrix> .ILinkable> outputVectors;
GetGraphOrder(out backwards, out _, out outputVectors);
while (outputVectors.Count != 0)
{
var node = outputVectors.Dequeue();
var inputVectors = backwards[node];
node.Data.state = new Matrix(node.Data.state.RowCount, node.Data.state.ColumnCount);
foreach (var inputVector in inputVectors)
{
var x = Training.AddBiasTerm(inputVector.a.Data.state);
node.Data.state += ((BaseMatrix)inputVector.b.Data.weights * (BaseMatrix)x);
}
node.Data.state = node.Data.type.Func(node.Data.state);
if (node.Secondary != null)
{
for (int i = 0; i < node.Data.size; i++)
{
CheckBox cb = (CheckBox)node.Secondary.Controls[i];
cb.Checked = node.Data.state[i, 0] >= 0.5 ? true : false;
}
}
}
}
private void button1_Click(object sender, EventArgs e)
{
var fun = new LogisticPerceptronFunc();
Matrix w = Matrix.Random(1, 3) * 0.5;
Thread t = new Thread(() => {
double learningrate = 1;
Matrix lastdw = Matrix.Ones(1, 3);
for (int i = 0; i < 10000; i++)
{
var data = binaryMapControl1.Data;
binaryMapControl1.w0 = (float)w[0, 0] * 1000;
binaryMapControl1.w1 = (float)w[0, 1];
binaryMapControl1.w2 = (float)w[0, 2];
List <Point> supposedClass1 = new List <Point>();
List <Point> supposedClass2 = new List <Point>();
foreach (var datum in data)
{
var x = Training.AddBiasTerm(datum.Item1 * 0.001);
// forward prop
var y = fun.Func((BaseMatrix)w * (BaseMatrix)x);
if (y[0, 0] > 0.5)
{
supposedClass1.Add(new Point((int)datum.Item1[0], (int)datum.Item1[1]));
}
else
{
supposedClass2.Add(new Point((int)datum.Item1[0], (int)datum.Item1[1]));
}
}
Matrix dwtotal = new Matrix(w.RowCount, w.ColumnCount);
foreach (var datum in data)
{
var x = Training.AddBiasTerm(datum.Item1 * 0.001);
// forward prop
var y = fun.Func((BaseMatrix)w * (BaseMatrix)x);
var dy = datum.Item2 - y;
Matrix dh;
Matrix dw;
Training.BackpropLayer(dy, x, w, fun, out dh, out dw);
dwtotal -= dw;
}
Thread.Sleep(10);
w = (w + dwtotal * (1 / (double)data.Count) * learningrate);
double change = dwtotal.ElementsRMS();
learningrate = (change == 0 ? 0 : 1 / change);
Invoke(new Action(() => {
binaryMapControl1.ShowOverlay(supposedClass1, supposedClass2);
matrixView1.Matrix(w);
matrixView2.Matrix(dwtotal);
}));
lastdw = dwtotal;
}
});
t.Start();
}
private void TrainFor(int iterations, TrainingType type, double learningRate, Action <double> callbackWithError)
{
OneToManyMap <GraphMap <FeatureVector, WeightMatrix> .ILinkable, Pair <GraphMap <FeatureVector, WeightMatrix> .ILinkable, GraphMap <FeatureVector, WeightMatrix> .Link <WeightMatrix> > > backwards;
Queue <GraphMap <FeatureVector, WeightMatrix> .ILinkable> inputVectors;
Queue <GraphMap <FeatureVector, WeightMatrix> .ILinkable> outputVectors;
GetGraphOrder(out backwards, out inputVectors, out outputVectors);
List <Dictionary <string, FeatureVector> > trainingData = new List <Dictionary <string, FeatureVector> >();
// Get training data
foreach (DataGridViewRow row in data_training.Rows)
{
string config = row.Cells[2].Value.ToString();
trainingData.Add(StateFromConfiguration(config).ToDictionary(x => x.name));
}
int iterationCounter = 0;
double averager = 1.0 / trainingData.Count;
double learningrate = learningRate;
// begin training...
while (true)
{
double squaredTrainingError = 0;
int totalTrainingNeuronCount = 0;
// Matrix weight gradients
Dictionary <GraphMap <FeatureVector, WeightMatrix> .Link <WeightMatrix>, Matrix> dw = new Dictionary <GraphMap <FeatureVector, WeightMatrix> .Link <WeightMatrix>, Matrix>();
foreach (var inputVector in inputVectors)
{
foreach (var edge in inputVector.Edges)
{
// make space to store the weight gradients
dw[edge.Value] = new Matrix(edge.Value.Data.weights.RowCount, edge.Value.Data.weights.ColumnCount);
}
}
foreach (var trainingCase in trainingData)
{
double perTrainingSquaredError = 0;
int perTrainingNeuronCount = 0;
// errors in all input vectors
Dictionary <string, Matrix> dy = new Dictionary <string, Matrix>();
// set all feature vectors to a training case
foreach (var feature in trainingCase)
{
if (vectors[feature.Key].Data.layer != LayerType.OUTPUT)
{
vectors[feature.Key].Data.state = feature.Value.state;
}
}
// forward prop
foreach (var outputVector in outputVectors)
{
var sources = backwards[outputVector];
outputVector.Data.state = new Matrix(outputVector.Data.state.RowCount, outputVector.Data.state.ColumnCount);
foreach (var source in sources)
{
var x = Training.AddBiasTerm(source.a.Data.state);
outputVector.Data.state += ((BaseMatrix)source.b.Data.weights * (BaseMatrix)x);
}
outputVector.Data.state = outputVector.Data.type.Func(outputVector.Data.state);
}
// Calculate errors
foreach (var output in outputs)
{
dy[output.Data.name] = -(trainingCase[output.Data.name].state - output.Data.state);
for (int i = 0; i < output.Data.state.RowCount; i++)
{
double error = dy[output.Data.name][i, 0];
perTrainingSquaredError += error * error;
perTrainingNeuronCount++;
}
}
squaredTrainingError += perTrainingSquaredError;
totalTrainingNeuronCount += perTrainingNeuronCount;
// Establish space for the input vectors
foreach (var inputVec in inputVectors)
{
dy[inputVec.Data.name] = new Matrix(inputVec.Data.state.RowCount, inputVec.Data.state.ColumnCount);
}
// backprop and add to weight gradients
foreach (var inputVec in inputVectors)
{
foreach (var edge in inputVec.Edges)
{
Matrix dHidden;
Matrix dWeights;
var x = Training.AddBiasTerm(inputVec.Data.state);
GraphMap <FeatureVector, WeightMatrix> .ILinkable asd;
asd = edge.Key;
FeatureVector data = asd.Data;
Training.BackpropLayer(dy[data.name], x, edge.Value.Data.weights, data.type, out dHidden, out dWeights);
dy[inputVec.Data.name] += Training.RemoveBiasTerm(dHidden);
if (type == TrainingType.Batch)
{
dw[edge.Value] -= dWeights;
}
else if (type == TrainingType.Online)
{
dw[edge.Value] = -dWeights;
}
}
}
// update weights
if (type == TrainingType.Online)
{
foreach (var inputVec in inputVectors)
{
foreach (var edge in inputVec.Edges)
{
edge.Value.Data.weights = (edge.Value.Data.weights + dw[edge.Value] * learningrate);
}
}
}
}
if (type == TrainingType.Batch)
{
// update weights
foreach (var inputVec in inputVectors)
{
foreach (var edge in inputVec.Edges)
{
edge.Value.Data.weights = (edge.Value.Data.weights + dw[edge.Value] * averager * learningrate);
}
}
}
// calculate total error
double totalError = Math.Sqrt(squaredTrainingError) / totalTrainingNeuronCount;
callbackWithError(totalError);
//Debug.WriteLine(totalError);
iterationCounter++;
// repeat until stopped
if (iterationCounter == iterations)
{
break;
}
}
}