public IWeightTensor Step(IWeightTensor input, IComputeGraph g)
{
using (IComputeGraph innerGraph = g.CreateSubGraph(m_name))
{
IWeightTensor hidden_prev = m_hidden;
IWeightTensor cell_prev = m_cell;
IWeightTensor inputs = innerGraph.Concate(1, input, hidden_prev);
IWeightTensor hhSum = innerGraph.Affine(inputs, m_Wxh, m_b);
IWeightTensor hhSum2 = m_layerNorm1.Norm(hhSum, innerGraph);
(IWeightTensor gates_raw, IWeightTensor cell_write_raw) = innerGraph.SplitColumns(hhSum2, m_hdim * 3, m_hdim);
IWeightTensor gates = innerGraph.Sigmoid(gates_raw);
IWeightTensor cell_write = innerGraph.Tanh(cell_write_raw);
(IWeightTensor input_gate, IWeightTensor forget_gate, IWeightTensor output_gate) = innerGraph.SplitColumns(gates, m_hdim, m_hdim, m_hdim);
// compute new cell activation: ct = forget_gate * cell_prev + input_gate * cell_write
m_cell = g.EltMulMulAdd(forget_gate, cell_prev, input_gate, cell_write);
IWeightTensor ct2 = m_layerNorm2.Norm(m_cell, innerGraph);
// compute hidden state as gated, saturated cell activations
m_hidden = g.EltMul(output_gate, innerGraph.Tanh(ct2));
return(m_hidden);
}
}
/// <summary>
/// Update LSTM-Attention cells according to given weights
/// </summary>
/// <param name="context">The context weights for attention</param>
/// <param name="input">The input weights</param>
/// <param name="computeGraph">The compute graph to build workflow</param>
/// <returns>Update hidden weights</returns>
public IWeightMatrix Step(IWeightMatrix context, IWeightMatrix input, IComputeGraph computeGraph)
{
var cell_prev = ct;
var hidden_prev = ht;
var hxhc = computeGraph.ConcatColumns(input, hidden_prev, context);
var bs = computeGraph.RepeatRows(b, input.Rows);
var hhSum = computeGraph.MulAdd(hxhc, Wxhc, bs);
(var gates_raw, var cell_write_raw) = computeGraph.SplitColumns(hhSum, hdim * 3, hdim);
var gates = computeGraph.Sigmoid(gates_raw);
var cell_write = computeGraph.Tanh(cell_write_raw);
(var input_gate, var forget_gate, var output_gate) = computeGraph.SplitColumns(gates, hdim, hdim, hdim);
// compute new cell activation
//var retain_cell = computeGraph.EltMul(forget_gate, cell_prev);
//var write_cell = computeGraph.EltMul(input_gate, cell_write);
//ct = computeGraph.Add(retain_cell, write_cell);
ct = computeGraph.EltMulMulAdd(forget_gate, cell_prev, input_gate, cell_write);
ht = computeGraph.EltMul(output_gate, computeGraph.Tanh(ct));
return(ht);
}
/// <summary>
/// Update LSTM-Attention cells according to given weights
/// </summary>
/// <param name="context">The context weights for attention</param>
/// <param name="input">The input weights</param>
/// <param name="computeGraph">The compute graph to build workflow</param>
/// <returns>Update hidden weights</returns>
public IWeightTensor Step(IWeightTensor context, IWeightTensor input, IComputeGraph g)
{
using (IComputeGraph computeGraph = g.CreateSubGraph(m_name))
{
IWeightTensor cell_prev = Cell;
IWeightTensor hidden_prev = Hidden;
IWeightTensor hxhc = computeGraph.Concate(1, input, hidden_prev, context);
IWeightTensor hhSum = computeGraph.Affine(hxhc, m_Wxhc, m_b);
IWeightTensor hhSum2 = m_layerNorm1.Norm(hhSum, computeGraph);
(IWeightTensor gates_raw, IWeightTensor cell_write_raw) = computeGraph.SplitColumns(hhSum2, m_hiddenDim * 3, m_hiddenDim);
IWeightTensor gates = computeGraph.Sigmoid(gates_raw);
IWeightTensor cell_write = computeGraph.Tanh(cell_write_raw);
(IWeightTensor input_gate, IWeightTensor forget_gate, IWeightTensor output_gate) = computeGraph.SplitColumns(gates, m_hiddenDim, m_hiddenDim, m_hiddenDim);
// compute new cell activation: ct = forget_gate * cell_prev + input_gate * cell_write
Cell = g.EltMulMulAdd(forget_gate, cell_prev, input_gate, cell_write);
IWeightTensor ct2 = m_layerNorm2.Norm(Cell, computeGraph);
Hidden = g.EltMul(output_gate, computeGraph.Tanh(ct2));
return(Hidden);
}
}
public IWeightMatrix Step(IWeightMatrix input, IComputeGraph innerGraph)
{
var hidden_prev = ht;
var cell_prev = ct;
var inputs = innerGraph.ConcatColumns(input, hidden_prev);
var bs = innerGraph.RepeatRows(b, input.Rows);
var hhSum = innerGraph.MulAdd(inputs, Wxh, bs);
(var gates_raw, var cell_write_raw) = innerGraph.SplitColumns(hhSum, hdim * 3, hdim);
var gates = innerGraph.Sigmoid(gates_raw);
var cell_write = innerGraph.Tanh(cell_write_raw);
(var input_gate, var forget_gate, var output_gate) = innerGraph.SplitColumns(gates, hdim, hdim, hdim);
// compute new cell activation
//var retain_cell = innerGraph.EltMul(forget_gate, cell_prev); // what do we keep from cell
//var write_cell = innerGraph.EltMul(input_gate, cell_write); // what do we write to cell
//ct = innerGraph.Add(retain_cell, write_cell); // new cell contents
ct = innerGraph.EltMulMulAdd(forget_gate, cell_prev, input_gate, cell_write);
// compute hidden state as gated, saturated cell activations
ht = innerGraph.EltMul(output_gate, innerGraph.Tanh(ct));
return(ht);
}
public IWeightTensor Step(IWeightTensor input, IComputeGraph g)
{
using (var innerGraph = g.CreateSubGraph(this.m_name))
{
var hidden_prev = this.m_hidden;
var cell_prev = this.m_cell;
var inputs = innerGraph.ConcatColumns(input, hidden_prev);
var hhSum = innerGraph.Affine(inputs, this.m_Wxh, this.m_b);
var hhSum2 = this.m_layerNorm1.Norm(hhSum, innerGraph);
var(gates_raw, cell_write_raw) = innerGraph.SplitColumns(hhSum2, this.m_hdim * 3, this.m_hdim);
var gates = innerGraph.Sigmoid(gates_raw);
var cell_write = innerGraph.Tanh(cell_write_raw);
var(input_gate, forget_gate, output_gate) = innerGraph.SplitColumns(gates, this.m_hdim, this.m_hdim, this.m_hdim);
// compute new cell activation: ct = forget_gate * cell_prev + input_gate * cell_write
this.m_cell = g.EltMulMulAdd(forget_gate, cell_prev, input_gate, cell_write);
var ct2 = this.m_layerNorm2.Norm(this.m_cell, innerGraph);
// compute hidden state as gated, saturated cell activations
this.m_hidden = g.EltMul(output_gate, innerGraph.Tanh(ct2));
return(this.m_hidden);
}
}
public IWeightMatrix Step(IWeightMatrix input, IComputeGraph innerGraph)
{
var hidden_prev = ht;
var cell_prev = ct;
var inputs = innerGraph.ConcatColumns(input, hidden_prev);
var bs = innerGraph.RepeatRows(b, input.Rows);
var hhSum = innerGraph.MulAdd(inputs, Wxh, bs);
var hhSum2 = layerNorm1.Process(hhSum, innerGraph);
(var gates_raw, var cell_write_raw) = innerGraph.SplitColumns(hhSum2, hdim * 3, hdim);
var gates = innerGraph.Sigmoid(gates_raw);
var cell_write = innerGraph.Tanh(cell_write_raw);
(var input_gate, var forget_gate, var output_gate) = innerGraph.SplitColumns(gates, hdim, hdim, hdim);
// compute new cell activation: ct = forget_gate * cell_prev + input_gate * cell_write
ct = innerGraph.EltMulMulAdd(forget_gate, cell_prev, input_gate, cell_write);
var ct2 = layerNorm2.Process(ct, innerGraph);
// compute hidden state as gated, saturated cell activations
ht = innerGraph.EltMul(output_gate, innerGraph.Tanh(ct2));
return(ht);
}
/// <summary>
/// Update LSTM-Attention cells according to given weights
/// </summary>
/// <param name="context">The context weights for attention</param>
/// <param name="input">The input weights</param>
/// <param name="computeGraph">The compute graph to build workflow</param>
/// <returns>Update hidden weights</returns>
public IWeightTensor Step(IWeightTensor context, IWeightTensor input, IComputeGraph g)
{
using (var computeGraph = g.CreateSubGraph(this.m_name))
{
var cell_prev = this.Cell;
var hidden_prev = this.Hidden;
var hxhc = computeGraph.ConcatColumns(input, hidden_prev, context);
var hhSum = computeGraph.Affine(hxhc, this.m_Wxhc, this.m_b);
var hhSum2 = this.m_layerNorm1.Norm(hhSum, computeGraph);
var(gates_raw, cell_write_raw) = computeGraph.SplitColumns(hhSum2, this.m_hiddenDim * 3, this.m_hiddenDim);
var gates = computeGraph.Sigmoid(gates_raw);
var cell_write = computeGraph.Tanh(cell_write_raw);
var(input_gate, forget_gate, output_gate) = computeGraph.SplitColumns(gates, this.m_hiddenDim, this.m_hiddenDim, this.m_hiddenDim);
// compute new cell activation: ct = forget_gate * cell_prev + input_gate * cell_write
this.Cell = g.EltMulMulAdd(forget_gate, cell_prev, input_gate, cell_write);
var ct2 = this.m_layerNorm2.Norm(this.Cell, computeGraph);
this.Hidden = g.EltMul(output_gate, computeGraph.Tanh(ct2));
return(this.Hidden);
}
}
/// <summary>
/// Update LSTM-Attention cells according to given weights
/// </summary>
/// <param name="context">The context weights for attention</param>
/// <param name="input">The input weights</param>
/// <param name="computeGraph">The compute graph to build workflow</param>
/// <returns>Update hidden weights</returns>
public IWeightMatrix Step(IWeightMatrix context, IWeightMatrix input, IComputeGraph computeGraph)
{
var cell_prev = ct;
var hidden_prev = ht;
var hxhc = computeGraph.ConcatColumns(input, hidden_prev, context);
var bs = computeGraph.RepeatRows(b, input.Rows);
var hhSum = computeGraph.MulAdd(hxhc, Wxhc, bs);
var hhSum2 = layerNorm1.Process(hhSum, computeGraph);
(var gates_raw, var cell_write_raw) = computeGraph.SplitColumns(hhSum2, hdim * 3, hdim);
var gates = computeGraph.Sigmoid(gates_raw);
var cell_write = computeGraph.Tanh(cell_write_raw);
(var input_gate, var forget_gate, var output_gate) = computeGraph.SplitColumns(gates, hdim, hdim, hdim);
// compute new cell activation: ct = forget_gate * cell_prev + input_gate * cell_write
ct = computeGraph.EltMulMulAdd(forget_gate, cell_prev, input_gate, cell_write);
var ct2 = layerNorm2.Process(ct, computeGraph);
ht = computeGraph.EltMul(output_gate, computeGraph.Tanh(ct2));
return(ht);
}