protected internal override ComputationBlock GetBackwardComputationBlock(BackwardComputationMode mode)
{
var block = new ComputationBlock();
string derivate;
if (IsRecurrent)
{
derivate = derivateValue.Ref;
}
else
{
string outputValue = LowerConnections[0].InputValue.Ref; // Because this is an op. node
derivate = ActivationFunction.Derivate(block, outputValue);
}
var errorSb = new StringBuilder();
foreach (NeuralConnection outputConn in LowerConnections)
{
if (errorSb.Length != 0) errorSb.Append("+");
errorSb.Append("(" + outputConn.CreateErrorValueExpression() + ")");
}
string error = "((" + errorSb.ToString() + ")*(" + derivate + "))";
block.Add("double bwError=" + error);
// Set Bias:
if (mode == BackwardComputationMode.FeedForward)
{
block.Add(gradientValue.Ref + "=bwError");
block.Add(gradientSumValue.Ref + "+=bwError");
}
else if (mode == BackwardComputationMode.Recurrent)
{
block.Add(gradientValue.Ref + "+=bwError");
}
else // BackwardComputationMode.RecurrentLastStep
{
block.Add(gradientValue.Ref + "+=bwError");
block.Add(gradientSumValue.Ref + "+=" + gradientValue.Ref);
}
// Set Input Connections:
block.Add("double igv");
foreach (NeuralConnection inputConn in UpperConnections)
{
// Set Error:
block.Add(inputConn.ErrorValue.Ref + "=bwError");
// Set Gradinets
block.Add("igv=(bwError*" + inputConn.CreateInputValueExpression() + ")");
if (mode == BackwardComputationMode.FeedForward)
{
block.Add(inputConn.GradientValue.Ref + "=igv");
block.Add(inputConn.GradientSumValue.Ref + "+=igv");
}
else if (mode == BackwardComputationMode.Recurrent)
{
block.Add(inputConn.GradientValue.Ref + "+=igv");
}
else // BackwardComputationMode.RecurrentLastStep
{
block.Add(inputConn.GradientValue.Ref + "+=igv");
block.Add(inputConn.GradientSumValue.Ref + "+=" + inputConn.GradientValue.Ref);
}
}
return block;
}
protected internal override void DefinePushForwardInformation(ComputationBlock block, string theStack)
{
base.DefinePushForwardInformation(block, theStack); // Push Inputs and Co.
block.Add(theStack + ".Push(" + derivateValue.Ref + ")");
}
protected internal override void DefinePopForwardInformation(ComputationBlock block, string theStack)
{
block.Add(derivateValue.Ref + "=" + theStack + ".Pop()");
base.DefinePopForwardInformation(block, theStack); // Pop Inputs and Co.
}
public void Add(ComputationBlock block)
{
Contract.Requires(block != null);
foreach (var r in block.References)
{
References.Add(r);
}
foreach (var ns in block.Namespaces)
{
Namespaces.Add(ns);
}
CodeBuilder.Append(block.CodeBuilder);
}
protected override ComputationBlock CreateComputationBlock()
{
var block = new ComputationBlock();
string sum = biasValue.Ref;
foreach (var conn in this.UpperConnections)
{
sum = sum + "+(" + conn.CreateComputationExpression() + ")";
}
block.Add("double anSum=" + sum);
if (IsBackwardInitialized)
{
block.Add("double anResult=" + ActivationFunction.Function(block, "anSum"));
string result = "anResult";
block.Add(LowerConnections[0].InputValue.Ref + "=" + result);
if (IsRecurrent)
{
Debug.Assert(derivateValue != null);
block.Add(derivateValue.Ref + "=" + ActivationFunction.Derivate(block, result));
}
}
else
{
block.Add(LowerConnections[0].InputValue.Ref + "=" + ActivationFunction.Function(block, "anSum"));
}
return block;
}
private void Initialize()
{
var network = Network;
if (network.IsRecurrent && network.IsBackwardComputationRequired)
{
int nodeCount = network.Nodes.Count;
var push = new ComputationBuilder<double>();
var pop = new ComputationBuilder<double>();
// Create stacks:
stacks = new Stack<double>[nodeCount];
for (int idx = 0; idx < nodeCount; idx++)
{
// Create stack:
stacks[idx] = new Stack<double>();
string theStackDef = "var theStack = ((Stack<double>[])context)[" + idx + "]";
string theStack = "theStack";
var pushBlock = new ComputationBlock(idx);
var popBlock = new ComputationBlock(idx);
pushBlock.Add(theStackDef);
pushBlock.AddReference(typeof(Stack<double>));
popBlock.Add(theStackDef);
popBlock.AddReference(typeof(Stack<double>));
network.Nodes[idx].DefinePushForwardInformation(pushBlock, theStack);
network.Nodes[idx].DefinePopForwardInformation(popBlock, theStack);
push.AddBlock(pushBlock);
pop.AddBlock(popBlock);
}
Parallel.Invoke(
() => pushForwardHandle = push.Compile(network.ValueSpace, "PushForwardValues", stacks, @"c:\Temp\neuroflow\PushForwardValues.cs"),
() => popForwardHandle = pop.Compile(network.ValueSpace, "PopForwardValues", stacks, @"c:\Temp\neuroflow\PopForwardValues.cs"));
}
}
public override string Derivate(ComputationBlock block, string value)
{
return Alpha.ToString();
}
public override string Function(ComputationBlock block, string value)
{
block.Add("double lafResult = ((" + value + ") * " + Alpha + ")");
block.Add("if (lafResult < -" + Alpha + ") lafResult = -" + Alpha + "; else if (lafResult > " + Alpha + ") lafResult = " + Alpha + ";");
return "lafResult";
}
public abstract string Derivate(ComputationBlock block, string value);
public abstract string Function(ComputationBlock block, string value);
