/// <summary>
/// <para>
/// Builds the layer by converting the abstract definition of the layer into
/// a concrete set of instructions for Tensorflow and a layer configuration
/// for use when training the model.
/// </para>
/// <para>This method should register any parameters and initializers with the compilation context.
/// So that they can be used during the training phase. </para>
/// <para>Additionally you are required to store the layer configuration in the
/// <see cref="context"/> property. This information is required as metadata
/// when the model is used.</para>
/// </summary>
/// <param name="context">Use this context to register trainable parameters
/// and build the computational graph for the layer</param>
public override TFOutput Compile(ModelCompilationContext context)
{
if (Configuration != null)
{
return(Configuration.Output);
}
var input = _input.Compile(context);
var inputDimension = _input.OutputShape[_input.OutputShape.Length - 1];
using (var scope = context.Graph.WithScope(Name))
{
TFShape weightsShape = new TFShape(inputDimension, _units);
TFShape biasShape = new TFShape(_units);
var weights = context.Graph.VariableV2(
weightsShape,
TFDataType.Double, operName: "Weights");
var initializers = new List <TFOperation>
{
context.Graph.Assign(weights, _weightsInitializer.Compile(context.Graph, weightsShape)).Operation
};
var parameters = new List <TFOutput>
{
weights
};
context.AddParameters(weights);
var output = context.Graph.MatMul(input, weights);
if (_useBias)
{
var bias = context.Graph.VariableV2(
biasShape,
TFDataType.Double, operName: "Bias");
initializers.Add(context.Graph.Assign(bias,
_biasInitializer.Compile(context.Graph, biasShape)).Operation);
parameters.Add(bias);
output = context.Graph.Add(output, bias);
}
output = _activation.Compile(context, output);
Configuration = new LayerConfiguration(parameters, initializers, output);
context.AddInitializers(initializers);
return(output);
}
}
public void ReturnsGraphElement(Type functionType)
{
var activation = (ActivationFunction)Activator.CreateInstance(functionType);
var context = new ModelCompilationContext(new TFGraph());
var input = context.Graph.Placeholder(TFDataType.Double, new TFShape(10, 10));
var output = activation.Compile(context, input);
output.Should().NotBeNull();
}
public void OutputShapeIsEqualToInputLayerShape()
{
var context = new ModelCompilationContext(new TFGraph());
var input = new Input(new [] { 10L });
var layer = new Dropout(0.2, input);
layer.Compile(context);
layer.OutputShape.Should().BeEquivalentTo(input.OutputShape);
}
public void ReturnsLossFunction()
{
var graph = new TFGraph();
var context = new ModelCompilationContext(graph);
var predictions = graph.Placeholder(TFDataType.Double, new TFShape(-1, 10));
var actuals = graph.Placeholder(TFDataType.Double, new TFShape(-1, 10));
var loss = new MeanSquaredError().Compile(context, predictions, actuals);
loss.Should().NotBeNull();
}
public void ShouldCompile()
{
var graph = new TFGraph();
var context = new ModelCompilationContext(graph);
var predictions = graph.Placeholder(TFDataType.Double, new TFShape(-1, 10));
var actuals = graph.Placeholder(TFDataType.Double, new TFShape(-1, 10));
var loss = new CategoricalCrossEntropy().Compile(context, predictions, actuals);
loss.Should().NotBeNull();
}
/// <summary>
/// <para>
/// Builds the layer by converting the abstract definition of the layer into
/// a concrete set of instructions for Tensorflow and a layer configuration
/// for use when training the model.
/// </para>
/// <para>This method should register any parameters and initializers with the compilation context.
/// So that they can be used during the training phase. </para>
/// <para>Additionally you are required to store the layer configuration in the
/// <see cref="Layer.Configuration"/> property. This information is required as metadata
/// when the model is used.</para>
/// <param name="context">Use this context to register trainable parameters
/// and build the computational graph for the layer</param>
public override TFOutput Compile(ModelCompilationContext context)
{
var inputLayer = _input.Compile(context);
var keepProb = context.Graph.Const(_rate);
var output = context.Graph.Dropout(inputLayer, keepProb,
context.Graph.GetTensorShape(inputLayer), _seed);
Configuration = new LayerConfiguration(new TFOutput[] { }, new TFOperation[] { }, output);
return(output);
}
/// <summary>
/// Compiles the loss function
/// </summary>
/// <param name="context">Compilation context to use</param>
/// <param name="predictions">The output of the graph producing predictions</param>
/// <param name="targets">The output of the graph containing the targets</param>
/// <returns>Returns the compiled loss function</returns>
public override TFOutput Compile(ModelCompilationContext context, TFOutput predictions, TFOutput targets)
{
// Formula: loss = -sum(log(pred) * target)
var graph = context.Graph;
// Values should be clipped to be between epsilon and 1-epsilon
// to prevent arithmic overflows into infinity.
var clipped = graph.ClipByValue(predictions,
graph.Const(_epsilon),
graph.Const(1 - _epsilon));
return(graph.Neg(graph.ReduceSum(graph.Mul(targets, graph.Log(clipped)))));
}
public void CreatesLayerConfigurationDuringCompilation()
{
var context = new ModelCompilationContext(new TFGraph());
var input = new Input(new [] { 10L });
var layer = new Dropout(0.2, input);
layer.Compile(context);
layer.Configuration.Should().NotBeNull();
layer.Configuration.Parameters.Count().Should().Be(0);
layer.Configuration.Initializers.Count().Should().Be(0);
layer.Configuration.Output.Should().NotBeNull();
}
public void ProducesHighLossForIncorrectCases()
{
var graph = new TFGraph();
var context = new ModelCompilationContext(graph);
var output = graph.Const(new[] { new[] { 0.0, 0.1 } });
var targets = graph.Const(new[] { new[] { 1.0, 0.0 } });
var loss = new NegativeLogLikelihood();
var lossFunction = loss.Compile(context, output, targets);
using (var session = new TFSession(graph))
{
var error = session.GetRunner().Run(lossFunction);
Math.Round((double)error.GetValue(), 2).Should().Be(16.12);
}
}
public void ProducesOutput(Type functionType)
{
var activation = (ActivationFunction)Activator.CreateInstance(functionType);
var graph = new TFGraph();
var context = new ModelCompilationContext(graph);
var input = context.Graph.Placeholder(TFDataType.Double, new TFShape(1));
var output = activation.Compile(context, input);
using (var session = new TFSession(graph))
{
var runner = session.GetRunner();
runner.AddInput(input, new TFTensor(new[] { new[] { 1.0 } }));
var outputValue = runner.Run(output);
outputValue.GetValue().Should().NotBe(new[] { new[] { 0.0 } });
}
}
/// <summary>
/// <para>
/// Builds the layer by converting the abstract definition of the layer into
/// a concrete set of instructions for Tensorflow and a layer configuration
/// for use when training the model.
/// </para>
/// <para>This method should register any parameters and initializers with the compilation context.
/// So that they can be used during the training phase. </para>
/// <para>Additionally you are required to store the layer configuration in the
/// <see cref="Configuration"/> property. This information is required as metadata
/// when the model is used.</para>
/// <param name="context">Use this context to register trainable parameters
/// and build the computational graph for the layer</param>
public override TFOutput Compile(ModelCompilationContext context)
{
if (Configuration != null)
{
return(Configuration.Output);
}
if (_shape.Length == 0)
{
throw new ModelCompilationException("Shape must have at least one dimension");
}
var placeholder = context.Graph.Placeholder(TFDataType.Double,
new TFShape(OutputShape), operName: Name);
Configuration = new LayerConfiguration(new TFOutput[] { }, new TFOperation[] { }, placeholder);
return(placeholder);
}
public void CanBeOptimized()
{
var graph = new TFGraph();
var context = new ModelCompilationContext(graph);
var input = new Input(new long[] { 10 }, name: "Input0");
var output = new Dense(2, input, name: "Dense0");
var compiledOutput = output.Compile(context);
var loss = new NegativeLogLikelihood();
var compiledLoss = loss.Compile(context, compiledOutput,
context.Graph.Placeholder(TFDataType.Double, new TFShape(-1, 2)));
var gradients = graph.AddGradients(
new[] { compiledLoss },
context.Parameters.ToArray());
}
public void ShouldBeOptimizable()
{
var graph = new TFGraph();
var context = new ModelCompilationContext(graph);
var input = new Input(new long[] { 10 }, name: "Input0");
var output = new Dense(2, input, name: "Dense0");
var compiledInput = input.Compile(context);
var compiledOutput = output.Compile(context);
var loss = new CategoricalCrossEntropy();
var compiledLoss = loss.Compile(context, compiledOutput,
context.Graph.Placeholder(TFDataType.Double, new TFShape(-1, 2)));
var gradients = graph.AddGradients(
new [] { compiledLoss },
context.Parameters.ToArray());
}
/// <summary>
/// Compiles the optimizer
/// </summary>
/// <param name="graph">Graph to use for compilation</param>
/// <param name="loss">Loss function to use</param>
/// <param name="parameters">Parameters to optimize</param>
public override void Compile(ModelCompilationContext context, TFOutput loss, IEnumerable <TFOutput> parameters)
{
var graph = context.Graph;
var operations = new List <TFOperation>();
var moments = new List <TFOutput>();
using (var optimizerScope = graph.WithScope("SGD"))
{
using (var momentScope = graph.WithScope("Moments"))
{
foreach (var parameter in parameters)
{
var moment = graph.VariableV2(graph.GetTensorShape(parameter), TFDataType.Double);
var initializer = graph.Assign(moment, graph.Zeros(graph.GetTensorShape(parameter))).Operation;
context.AddInitializers(initializer);
moments.Add(moment);
}
}
var momentum = graph.Const(_momentum, "Momentum");
var learningRate = graph.Const(_learningRate);
var gradients = graph.AddGradients(new[] { loss }, parameters.ToArray());
foreach (var(parameter, gradient, moment) in ZipLearningParameters(parameters, gradients, moments))
{
// velocity = momentum * moment - learningRate * gradient
var velocity = graph.Sub(graph.Mul(momentum, moment), graph.Mul(gradient, learningRate));
var newWeight = graph.Add(parameter, velocity);
var newMomentum = graph.Add(moment, velocity);
operations.Add(graph.Assign(parameter, newWeight).Operation);
operations.Add(graph.Assign(moment, newMomentum).Operation);
}
}
Operations = operations;
}
/// <summary> /// Compiles the optimizer /// </summary> /// <param name="context">Context to use for compilation</param> /// <param name="loss">Loss function to use</param> /// <param name="parameters">Parameters to optimize</param> public abstract void Compile(ModelCompilationContext context, TFOutput loss, IEnumerable <TFOutput> parameters);
/// <summary> /// Compiles the loss function /// </summary> /// <param name="context">Compilation context to use</param> /// <param name="predictions">The output of the graph producing predictions</param> /// <param name="targets">The output of the graph containing the targets</param> /// <returns>Returns the compiled loss function</returns> public abstract TFOutput Compile(ModelCompilationContext context, TFOutput predictions, TFOutput targets);
/// <summary>
/// Compiles the categorical cross entropy function
/// </summary>
/// <param name="context">Compilation context to use</param>
/// <param name="predictions">Output of the neural network</param>
/// <param name="targets">Targets to optimize towards</param>
/// <returns>Returns the compiled loss function</returns>
public override TFOutput Compile(ModelCompilationContext context, TFOutput predictions, TFOutput targets)
{
var(loss, backprop) = context.Graph.SoftmaxCrossEntropyWithLogits(predictions, targets);
return(loss);
}
/// <summary> /// <para> /// Builds the layer by converting the abstract definition of the layer into /// a concrete set of instructions for Tensorflow and a layer configuration /// for use when training the model. /// </para> /// <para>This method should register any parameters and initializers with the compilation context. /// So that they can be used during the training phase. </para> /// <para>Additionally you are required to store the layer configuration in the /// <see cref="Configuration"/> property. This information is required as metadata /// when the model is used.</para> /// <param name="context">Use this context to register trainable parameters /// and build the computational graph for the layer</param> public abstract TFOutput Compile(ModelCompilationContext context);
/// <summary>
/// Compiles the loss function
/// </summary>
/// <param name="context">Compilation context to use</param>
/// <param name="predictions">The output of the graph producing predictions</param>
/// <param name="targets">The output of the graph containing the targets</param>
/// <returns>Returns the compiled loss function</returns>
public override TFOutput Compile(ModelCompilationContext context, TFOutput predictions, TFOutput targets)
{
return(context.Graph.Mean(context.Graph.Square(context.Graph.Sub(predictions, targets)), context.Graph.Const(-1)));
}
/// <summary>
/// Compiles the activation function
/// </summary>
/// <param name="context">Use this context to register trainable parameters
/// and build the computational graph for the layer</param>
/// <param name="input">Input for the activation function</param>
/// <returns>Returns the compiled activation function</returns>
public override TFOutput Compile(ModelCompilationContext context, TFOutput input)
{
return(context.Graph.Relu(input));
}
