void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId)
{
int hiddenLayerSize = memory.Length;
_inputSize = inputSize;
_memory = new MemoryFeeder(memory, null, memoryId);
_input = new FlowThrough();
var Wx = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wx").LastNode;
var Wi = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wi");
var Ui = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui");
var Wf = graph.Connect(hiddenLayerSize, Wx).AddFeedForward(hiddenLayerSize, "Wf");
var Uf = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf");
var It = graph.Add(Wi, Ui).Add(graph.SigmoidActivation());
var Ft = graph.Add(Wf, Uf).Add(graph.SigmoidActivation());
_output = graph
.Add(graph.Multiply(hiddenLayerSize, Wx, It.LastNode), graph.Multiply(hiddenLayerSize, _memory, Ft.LastNode))
.AddForwardAction(_memory.SetMemoryAction)
//.Add(new HookErrorSignal(context => {
// if (_lastBackpropagation != null) {
// foreach (var item in _lastBackpropagation)
// context.AppendErrorSignal(item.Value, item.Key);
// _lastBackpropagation = null;
// }
//}))
.LastNode
;
_start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
}
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId)
{
_inputSize = inputSize;
int hiddenLayerSize = memory.Length;
_memory = new MemoryFeeder(memory, null, memoryId);
_state = new MemoryFeeder(new float[memory.Length]);
_input = new FlowThrough();
var Wf = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wf");
var Wi = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wi");
var Wo = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wo");
var Wc = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wc");
var Uf = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uf");
var Ui = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ui");
var Uo = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uo");
var Uc = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uc");
var Ft = graph.Add(Wf, Uf).Add(graph.SigmoidActivation("Ft"));
var It = graph.Add(Wi, Ui).Add(graph.SigmoidActivation("It"));
var Ot = graph.Add(Wo, Uo).Add(graph.SigmoidActivation("Ot"));
var ftCt1 = graph.Multiply(hiddenLayerSize, Ft.LastNode, _state);
var Ct = graph.Add(ftCt1, graph.Multiply(It, graph.Add(Wc, Uc).Add(graph.TanhActivation()))).
AddForwardAction(_state.SetMemoryAction);
_output = graph.Multiply(Ot, Ct.Add(graph.TanhActivation())).
AddForwardAction(_memory.SetMemoryAction)
//.Add(new HookErrorSignal(context => {
// if (_lastBackpropagation != null) {
// foreach (var item in _lastBackpropagation)
// context.AppendErrorSignal(item.Value, item.Key);
// _lastBackpropagation = null;
// }
//}))
.LastNode;
_start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
}
void _Create(GraphFactory graph, int inputSize, float[] memory, INode activation, string memoryId)
{
_inputSize = inputSize;
_activation = activation;
int hiddenLayerSize = memory.Length;
_memory = new MemoryFeeder(memory, null, memoryId);
_input = new FlowThrough();
var inputChannel = graph.Connect(inputSize, _input)
.AddFeedForward(hiddenLayerSize, "Wh");
var memoryChannel = graph.Connect(hiddenLayerSize, _memory)
.AddFeedForward(hiddenLayerSize, "Uh");
_output = graph.Add(inputChannel, memoryChannel)
.AddBackwardAction(new ConstrainSignal())
.Add(activation)
.AddForwardAction(_memory.SetMemoryAction)
//.Add(new HookErrorSignal(context => {
// if (_lastBackpropagation != null) {
// foreach (var item in _lastBackpropagation)
// context.AppendErrorSignal(item.Value, item.Key);
// _lastBackpropagation = null;
// }
//}))
.LastNode;
;
_start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
}
void _Create(GraphFactory graph, int inputSize, float[] memory, string memoryId)
{
_inputSize = inputSize;
int hiddenLayerSize = memory.Length;
_memory = new MemoryFeeder(memory, null, memoryId);
_input = new FlowThrough();
var Wz = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wz");
var Uz = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Uz");
var Wr = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wr");
var Ur = graph.Connect(hiddenLayerSize, _memory).AddFeedForward(hiddenLayerSize, "Ur");
// add sigmoids to the gates
var Rt = graph.Add(Wr, Ur).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Rt")).LastNode;
var Zt = graph.Add(Wz, Uz).AddBackwardAction(new ConstrainSignal()).Add(graph.SigmoidActivation("Zt")).LastNode;
// h1 = tanh(Wh(x) + Uh(Ht1xRt))
var Wh = graph.Connect(inputSize, _input).AddFeedForward(hiddenLayerSize, "Wh");
var Uh = graph.Multiply(hiddenLayerSize, Rt, _memory).AddFeedForward(hiddenLayerSize, "Uh");
var h1 = graph.Add(Wh, Uh).AddBackwardAction(new ConstrainSignal()).Add(graph.TanhActivation());
// h2 = h1x(1-Zt)
var h2 = graph.Multiply(h1, graph.Connect(hiddenLayerSize, Zt).Add(graph.CreateOneMinusInput()));
// h = h1xh2
var previous = graph.Multiply(hiddenLayerSize, Zt, _memory);
_output = graph
.Add(h2, previous)
.AddForwardAction(_memory.SetMemoryAction)
//.Add(new HookErrorSignal(context => {
// if (_lastBackpropagation != null) {
// foreach (var item in _lastBackpropagation)
// context.AppendErrorSignal(item.Value, item.Key);
// _lastBackpropagation = null;
// }
//}))
.LastNode
;
_start = new OneToMany(SubNodes, bp => _lastBackpropagation = bp);
}
void _Create(GraphFactory graph, int inputSize, float[] gamma, float[] beta, float[] derivedMean, float[] derivedM2, int count)
{
_inputSize = inputSize;
_statistics = new VectorBasedStatistics(graph.LinearAlgebraProvider, inputSize, derivedMean, derivedM2, count);
if (gamma == null)
{
gamma = new float[inputSize];
for (var i = 0; i < inputSize; i++)
{
gamma[i] = 1f;
}
}
if (beta == null)
{
beta = new float[inputSize];
}
_input = new FlowThrough();
_gamma = new VectorInput(gamma, "gamma");
_beta = new VectorInput(beta, "beta");
var mean = graph.Connect(inputSize, _input).Add(new BatchMean());
var subtractMean = graph.Subtract(inputSize, _input, mean.LastNode);
var subtractMeanOutput = subtractMean.LastNode;
var stdDev = subtractMean
.Add(new InputSquared())
.Add(new BatchMean())
.Add(new SquareRootOfInput())
.Add(new OneDividedByInput());
var normalised = graph.Multiply(_inputSize, subtractMeanOutput, stdDev.LastNode) /*.Add(new InspectSignals(data => {
* var matrix = data.GetMatrix().AsIndexable();
* var statistics = matrix.Columns.Select(v => (v, v.Average())).Select(v2 => (v2.Item2, v2.Item1.StdDev(v2.Item2))).ToList();
* }))*/;
var adjusted = graph.Multiply(_inputSize, _gamma, normalised.LastNode);
_output = graph.Add(_inputSize, _beta, adjusted.LastNode).LastNode;
_start = new OneToMany(SubNodes, bp => { });
}
void _Create(GraphFactory graph, float gamma, float beta)
{
_input = new FlowThrough();
_input2 = new CalculateVariance();
var main = graph.Subtract(_inputSize, _input, _input2);
var mainNode = main.LastNode;
main.Add(new InputSquared());
main.Add(new CalculateVariance());
main.Add(new SquareRootOfInput());
main.Add(new OneDividedByInput());
main = graph.Multiply(_inputSize, mainNode, main.LastNode);
_gamma = new MultiplyWithParameter(gamma);
_beta = new MultiplyWithParameter(beta);
main = graph.Multiply(_inputSize, _gamma, main.LastNode);
_output = graph.Add(_inputSize, _beta, main.LastNode).LastNode;
_start = new OneToMany(SubNodes, null);
}