/// <summary>
/// <para>
/// Builds the layer by converting the abstract definition of the layer into
/// a concrete set of instructions for Tensorflow and a layer configuration
/// for use when training the model.
/// </para>
/// <para>This method should register any parameters and initializers with the compilation context.
/// So that they can be used during the training phase. </para>
/// <para>Additionally you are required to store the layer configuration in the
/// <see cref="context"/> property. This information is required as metadata
/// when the model is used.</para>
/// </summary>
/// <param name="context">Use this context to register trainable parameters
/// and build the computational graph for the layer</param>
public override TFOutput Compile(ModelCompilationContext context)
{
if (Configuration != null)
{
return(Configuration.Output);
}
var input = _input.Compile(context);
var inputDimension = _input.OutputShape[_input.OutputShape.Length - 1];
using (var scope = context.Graph.WithScope(Name))
{
TFShape weightsShape = new TFShape(inputDimension, _units);
TFShape biasShape = new TFShape(_units);
var weights = context.Graph.VariableV2(
weightsShape,
TFDataType.Double, operName: "Weights");
var initializers = new List <TFOperation>
{
context.Graph.Assign(weights, _weightsInitializer.Compile(context.Graph, weightsShape)).Operation
};
var parameters = new List <TFOutput>
{
weights
};
context.AddParameters(weights);
var output = context.Graph.MatMul(input, weights);
if (_useBias)
{
var bias = context.Graph.VariableV2(
biasShape,
TFDataType.Double, operName: "Bias");
initializers.Add(context.Graph.Assign(bias,
_biasInitializer.Compile(context.Graph, biasShape)).Operation);
parameters.Add(bias);
output = context.Graph.Add(output, bias);
}
output = _activation.Compile(context, output);
Configuration = new LayerConfiguration(parameters, initializers, output);
context.AddInitializers(initializers);
return(output);
}
}
/// <summary>
/// Compiles the optimizer
/// </summary>
/// <param name="graph">Graph to use for compilation</param>
/// <param name="loss">Loss function to use</param>
/// <param name="parameters">Parameters to optimize</param>
public override void Compile(ModelCompilationContext context, TFOutput loss, IEnumerable <TFOutput> parameters)
{
var graph = context.Graph;
var operations = new List <TFOperation>();
var moments = new List <TFOutput>();
using (var optimizerScope = graph.WithScope("SGD"))
{
using (var momentScope = graph.WithScope("Moments"))
{
foreach (var parameter in parameters)
{
var moment = graph.VariableV2(graph.GetTensorShape(parameter), TFDataType.Double);
var initializer = graph.Assign(moment, graph.Zeros(graph.GetTensorShape(parameter))).Operation;
context.AddInitializers(initializer);
moments.Add(moment);
}
}
var momentum = graph.Const(_momentum, "Momentum");
var learningRate = graph.Const(_learningRate);
var gradients = graph.AddGradients(new[] { loss }, parameters.ToArray());
foreach (var(parameter, gradient, moment) in ZipLearningParameters(parameters, gradients, moments))
{
// velocity = momentum * moment - learningRate * gradient
var velocity = graph.Sub(graph.Mul(momentum, moment), graph.Mul(gradient, learningRate));
var newWeight = graph.Add(parameter, velocity);
var newMomentum = graph.Add(moment, velocity);
operations.Add(graph.Assign(parameter, newWeight).Operation);
operations.Add(graph.Assign(moment, newMomentum).Operation);
}
}
Operations = operations;
}