/// <summary>
/// Run this layer. Take relevant input values from inputs and put relevant output values in outputs registry. Each input and each output registry represents one connected layer.
/// </summary>
/// <param name="buffer">The buffer containing the inputs, parameters and outputs respective to this layer.</param>
/// <param name="handler">The computation handler to use for computations (duh).</param>
/// <param name="trainingPass">Indicate whether this is run is part of a training pass.</param>
public override void Run(ILayerBuffer buffer, IComputationHandler handler, bool trainingPass)
{
INDArray inputs = buffer.Inputs["default"].Get <INDArray>("activations");
INDArray weights = buffer.Parameters.Get <INDArray>("weights");
string activation = buffer.Parameters.Get <string>("activation");
long batches = inputs.Shape[0];
int inputSize = Parameters.Get <int>("default_input_size"), size = Parameters.Get <int>("size");
INDArray biases = handler.StackRows((int)batches, buffer.Parameters.Get <INDArray>("biases"));
INDArray activations = handler.PermuteBatchAndTime(inputs); // BatchTimeFeatures ordering by default, needs to be TimeBatchFeatures for layers operating on the time dimension
activations = activations.Reshape(activations.Shape[0], activations.Shape[1] * ArrayUtils.Product(2, activations.Shape));
activations = handler.RowWise(activations, timeSlice =>
{
timeSlice = timeSlice.Reshape(inputs.Shape[0], inputSize);
timeSlice = handler.Dot(timeSlice, weights);
timeSlice = handler.Add(timeSlice, biases);
timeSlice = handler.Activation(activation, timeSlice);
return(timeSlice.Reshape(1L, batches * size));
});
activations = activations.Reshape(activations.Shape[0], batches, size);
buffer.Outputs["default"]["activations"] = handler.PermuteBatchAndTime(activations); // TODO are those the right dimensions? they should be...
}
/// <inheritdoc />
public INDArray FlattenTime(INDArray array)
{
long[] newShape = new long[array.Shape.Length - 1];
newShape[0] = checked (array.Shape[0] * array.Shape[1]);
for (var i = 0; i < newShape.Length; i++)
{
newShape[i] = array.Shape[i + 1];
}
return(array.Reshape(newShape));
}
/// <inheritdoc />
public override void Run(ILayerBuffer buffer, IComputationHandler handler, bool trainingPass)
{
INDArray input = buffer.Inputs["default"].Get <INDArray>("activations");
INDArray activations = handler.FlattenTimeAndFeatures(input);
INDArray weights = buffer.Parameters.Get <INDArray>("weights");
INDArray biases = handler.StackRows((int)(input.Shape[0] * input.Shape[1]), buffer.Parameters.Get <INDArray>("biases"));
INDArray output = handler.Dot(activations, weights);
output = handler.Add(output, biases);
output = handler.Activation(buffer.Parameters.Get <string>("activation"), output);
buffer.Outputs["default"]["activations"] = output.Reshape(input.Shape[0], input.Shape[1], Parameters.Get <int>("size"));
}
/// <summary>
/// Run this layer. Take relevant input values from inputs and put relevant output values in outputs registry. Each input and each output registry represents one connected layer.
/// </summary>
/// <param name="buffer">The buffer containing the inputs, parameters and outputs respective to this layer.</param>
/// <param name="handler">The computation handler to use for computations (duh).</param>
/// <param name="trainingPass">Indicate whether this is run is part of a training pass.</param>
public override void Run(ILayerBuffer buffer, IComputationHandler handler, bool trainingPass)
{
if (trainingPass)
{
INDArray inputs = buffer.Inputs["default"].Get <INDArray>("activations");
INDArray dropoutMask = handler.NDArray((long[])inputs.Shape.Clone());
handler.FillWithProbabilityMask(dropoutMask, 1.0 - Parameters.Get <double>("dropout_probability"));
INDArray activations = handler.Multiply(inputs, dropoutMask);
buffer.Outputs["default"]["activations"] = activations.Reshape((long[])inputs.Shape.Clone());
}
else
{
buffer.Outputs["default"]["activations"] = buffer.Inputs["default"]["activations"];
}
}
/// <summary>
/// Invoke this hook with a certain parameter registry if optional conditional criteria are satisfied.
/// </summary>
/// <param name="registry">The registry containing the required values for this hook's execution.</param>
/// <param name="resolver">A helper resolver for complex registry entries (automatically cached).</param>
public override void SubInvoke(IRegistry registry, IRegistryResolver resolver)
{
// we need copies of network and optimiser as to not affect the current internal state
INetwork network = (INetwork)resolver.ResolveGetSingle <INetwork>("network.self").DeepCopy();
BaseGradientOptimiser optimiser = (BaseGradientOptimiser)resolver.ResolveGetSingle <BaseGradientOptimiser>("optimiser.self").ShallowCopy();
INDArray desiredTargets = ParameterRegistry.Get <INDArray>("desired_targets");
IComputationHandler handler = new DebugHandler(Operator.Handler);
long[] inputShape = network.YieldExternalInputsLayerBuffers().First().Parameters.Get <long[]>("shape");
IDictionary <string, INDArray> block = DataUtils.MakeBlock("targets", desiredTargets); // desired targets don't change during execution
double desiredCost = ParameterRegistry.Get <double>("desired_cost"), currentCost = Double.MaxValue;
int maxOptimisationAttempts = ParameterRegistry.Get <int>("max_optimisation_attempts");
int maxOptimisationSteps = ParameterRegistry.Get <int>("max_optimisation_steps");
int optimisationSteps = 0;
INDArray maximisedInputs = CreateRandomisedInput(handler, inputShape);
for (int i = 0; i < maxOptimisationAttempts; i++)
{
optimisationSteps = 0;
do
{
// trace current inputs and run network as normal
uint traceTag = handler.BeginTrace();
block["inputs"] = handler.Trace(maximisedInputs.Reshape(ArrayUtils.Concatenate(new[] { 1L, 1L }, inputShape)), traceTag);
handler.BeginSession();
DataUtils.ProvideExternalInputData(network, block);
network.Run(handler, trainingPass: false);
// fetch current outputs and optimise against them (towards desired targets)
INDArray currentTargets = network.YieldExternalOutputsLayerBuffers().First(b => b.ExternalOutputs.Contains("external_default"))
.Outputs["external_default"].Get <INDArray>("activations");
INumber squaredDifference = handler.Sum(handler.Pow(handler.Subtract(handler.FlattenTimeAndFeatures(currentTargets), desiredTargets), 2));
handler.ComputeDerivativesTo(squaredDifference);
handler.EndSession();
INDArray gradient = handler.GetDerivative(block["inputs"]);
maximisedInputs = handler.ClearTrace(optimiser.Optimise("inputs", block["inputs"], gradient, handler));
currentCost = squaredDifference.GetValueAs <double>();
if (currentCost <= desiredCost)
{
goto Validation;
}
} while (++optimisationSteps < maxOptimisationSteps);
maximisedInputs = CreateRandomisedInput(handler, inputShape); // reset input
}
Validation:
maximisedInputs.ReshapeSelf(inputShape);
string sharedResultInput = ParameterRegistry.Get <string>("shared_result_input_key");
string sharedResultSuccess = ParameterRegistry.Get <string>("shared_result_success_key");
if (optimisationSteps >= maxOptimisationSteps)
{
_logger.Debug($"Aborted target maximisation for {desiredTargets}, failed after {maxOptimisationSteps} optimisation steps in {maxOptimisationAttempts} attempts (exceeded limit, current cost {currentCost} but desired {desiredCost}).");
resolver.ResolveSet(sharedResultSuccess, false, addIdentifierIfNotExists: true);
}
else
{
_logger.Debug($"Successfully finished target optimisation for {desiredTargets} after {optimiser} optimisation steps.");
resolver.ResolveSet(sharedResultSuccess, true, addIdentifierIfNotExists: true);
resolver.ResolveSet(sharedResultInput, maximisedInputs, addIdentifierIfNotExists: true);
}
}
/// <inheritdoc />
public INDArray FlattenAllButLast(INDArray array)
{
return(array.Reshape(ArrayUtils.Product(0, array.Rank - 1, array.Shape), array.Shape[array.Rank - 1]));
}
/// <inheritdoc />
public INDArray FlattenTimeAndFeatures(INDArray array)
{
return(array.Reshape(array.Shape[0] * array.Shape[1], ArrayUtils.Product(2, array.Shape)));
}
