public void SoftmaxPropagateTest()
{
int nbInput = 10;
NNArray input = Utils.OneHot(nbInput, 5);
NNArray output = Utils.OneHot(nbInput, 3);
IActivation activation;
activation = new Softmax();
var network = new Network(
new IdentityLayer(nbInput),
new DenseLayerNoBias(nbInput, nbInput, activation, new SquaredDistance()));
network.Initialize();
DateTime start;
DateTime end;
int epoc = 0, maxEpoc = 10000;
double error = double.MaxValue;
start = DateTime.Now;
while (++epoc < maxEpoc && error > 0.05)
{
error = network.Train(input, output, 0.01);
}
end = DateTime.Now;
var duration = (end - start).TotalMilliseconds / 1000;
Console.WriteLine($"Duration for activation {activation.Name}: {duration} \t epoc: {epoc}\terror: {error}");
Assert.IsTrue(epoc < maxEpoc);
}
public void SoftmaxPrimeTest()
{
var a = new Matrix(4, 4);
a.InRandomize();
var sf = new Softmax();
// Jacobian Matrix
var cachedSoftmax = sf.Forward(a);
a.InFlatten();
var jac = new Matrix(a.Rows, a.Rows).Fill(0);
for (var i = 0; i < cachedSoftmax.Rows; i++)
{
jac[i, i] = a[i, 0];
}
for (var i = 0; i < jac.Rows; i++)
{
for (var j = 0; j < jac.Columns; j++)
{
if (i == j)
{
jac[i, j] = cachedSoftmax[i, 0] * (1 - cachedSoftmax[j, 0]);
}
}
}
var b = jac - a * a.T();
Assert.IsTrue(Math.Abs(sf.Backward(a).FrobeniusNorm() - b.FrobeniusNorm()) < 0.1, new Softmax().Type().ToString() + " Derivative.");
}
public void CrossEntropyTest()
{
var crossEntropyOH = new CrossEntropyOneHot();
var crossEntropy = new CrossEntropy();
int nb = 10;
double[] input = new double[nb];
double[] output = new double[nb];
for (int i = 0; i < nb; i++)
{
input[i] = Math.PI * rnd.NextDouble();
}
var softmax = new Softmax();
softmax.Activate(input, output);
double[] expectedOutput = new double[nb];
expectedOutput[output.ArgMax()] = 1;
double[] errors = new double[nb];
double entropyOH = crossEntropyOH.Evaluate(output, expectedOutput, errors);
double entropy = crossEntropy.Evaluate(output, expectedOutput, errors);
Assert.AreEqual(entropyOH, entropy);
}
public SoftmaxLayerArgument Convert(Softmax layer, ConvertContext context)
{
return(new SoftmaxLayerArgument
{
Channels = (uint)layer.Input.Dimensions[1]
});
}
static void Main(string[] args)
{
float[] grid = new float[] { 11, 12, 13, 14, 15, 16, 17, 18, 19 };
Softmax softmax = new Softmax();
var sss = softmax.Forward(new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 });
var ss2 = softmax.Backward(grid);
float[][] grid2 = new float[2][];
grid2[0] = new float[] { 11, 12, 13, 14, 15 };
grid2[1] = new float[] { 1, 1, 1, 0, 1 };
float[][] data = new float[2][];
data[0] = new float[] { 1, 2, 3, 4, 5 };
data[1] = new float[] { 6, 7, 8, 9, 10 };
Softmax softmax2 = new Softmax();
var ss = softmax2.Forward(data);
var s2 = softmax2.Backward(grid2);
float[,] grid3 = new float[2, 5] {
{ 11, 12, 13, 14, 15 }, { 1, 1, 1, 0, 1 }
};
float[,] data3 = new float[2, 5] {
{ 1, 2, 3, 4, 5 }, { 6, 7, 8, 9, 10 }
};
softmax2 = new Softmax();
var ss22 = softmax2.Forward(data3);
var s22 = softmax2.Backward(grid3);
}
public static Softmax Softmax(Layer input, int axis = 1)
{
var inputConn = input.Outputs.First().Value;
var node = new Softmax(inputConn.Dimensions, axis);
inputConn.Connect(node.Input);
return(node);
}
public void Infer(Softmax layer, SoftmaxLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.MainMemoryMap[layer.Input.Connection.From];
var outputAlloc = context.MainMemoryMap[layer.Output];
argument.Flags = K210LayerFlags.MainMemoryOutput;
argument.MainMemoryInputAddress = inputAlloc.GetAddress();
argument.MainMemoryOutputAddress = outputAlloc.GetAddress();
}
public double[] GetOutputSoftMax()
{
var returnValue = new List <double>();
foreach (Neuron neuron in Layers.Last().Neurons)
{
returnValue.Add(neuron.CalculateOutput());
}
return(Softmax.CalculateOutput(returnValue.ToArray()));
}
private Layer ConvertSoftmax(LayerParameter layerParam)
{
var input = _outputs[layerParam.Bottom[0]];
var param = layerParam.SoftmaxParam;
var layer = new Softmax(input.Dimensions);
layer.Input.SetConnection(input);
_outputs.Add(layerParam.Top[0], layer.Output);
return(layer);
}
private Layer ConvertSoftmax(paddle.OpDesc op)
{
var x = GetParameter(op.Inputs, "X").Arguments[0];
var output = GetParameter(op.Outputs, "Out").Arguments[0];
var layer = new Softmax(GetVarShape(x));
_inputs.Add(layer.Input, x);
_outputs.Add(output, layer.Output);
return(layer);
}
public void Softmax()
{
var x = new Const <double>(1.0, "x");
var op = new Softmax <double>(x);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Softmax <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(1, deserialized.Parents.Count);
Assert.AreEqual("x", (deserialized.Parents[0] as Const <double>).Name);
}
private Layer ConvertSoftmax(tflite.Operator op)
{
var inputs = op.GetInputsArray();
var input = _graph.Tensors(inputs[0]).Value;
var options = op.BuiltinOptions <tflite.SoftmaxOptions>().Value;
var layer = new Softmax(input.GetShapeArray().ToNCHW());
_inputs.Add(layer.Input, inputs[0]);
_outputs.Add(op.Outputs(0), layer.Output);
return(layer);
}
public void MNISTPropagateTest()
{
MnistReader.RootPath = Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, @"..\..\..\MNIST"));
var images = MnistReader.ReadTestData().ToList();
Assert.IsTrue(images.Count() > 0);
var image = images.ElementAt(0);
NNArray input = image.Values;
NNArray output = Utils.OneHot(10, image.Label);
var nbInput = input.Length;
IActivation activation;
activation = new Softmax();
var network = new Network(
new NormalizedLayer(nbInput, 1),
new DenseLayerNoBias(nbInput, 10, activation, new CrossEntropyOneHot()));
// network.AddLayer(new DenseLayerNoBias(nbInput, 28, activation, new SquaredDistance()));
network.Initialize();
DateTime start;
DateTime end;
int epoc = 0, maxEpoc = 10000;
double error = double.MaxValue;
start = DateTime.Now;
while (++epoc < maxEpoc && error > 0.01)
{
error = network.Train(input, output, 0.01);
}
end = DateTime.Now;
var duration = (end - start).TotalMilliseconds / 1000;
Console.WriteLine($"Duration for activation {activation.Name}: {duration} \t epoc: {epoc}\terror: {error}");
Assert.AreEqual(image.Label, network.OutputLayer.Output.ArgMax());
Assert.IsTrue(epoc < maxEpoc);
foreach (var img in images.Where(i => i.Label == image.Label))
{
network.Evaluate(img.Values);
Console.WriteLine($"{network.OutputLayer.Output.ArgMax()}");
}
}
public void SoftmaxTest()
{
var a = new Matrix(2, 2);
a.InRandomize();
var sumExp = a.Map(Math.Exp).Sum();
var res = a.Map(Math.Exp) / sumExp;
var y = new Softmax().Forward(a);
Assert.IsTrue(res.ToString() == y.ToString(), new Softmax().Type().ToString() + " Activation.");
}
private static Layer ParseSoftmaxNode(NodeProto node, Dictionary <string, TensorType> types, Dictionary <string, OutputConnector> outputConns)
{
var inputType = types[node.Input[0]];
var axis = node.Attribute.SingleOrDefault(o => o.Name == "axis");
var layer = new Softmax(inputType.Dimensions, (int?)axis?.I ?? 1)
{
Name = node.Name
};
outputConns[node.Input[0]].Connect(layer.Input);
types.Add(node.Output[0], new TensorType {
ElementType = typeof(double), Dimensions = layer.Output.Dimensions.ToArray()
});
outputConns.Add(node.Output[0], layer.Output);
return(layer);
}
public void SoftMax()
{
int nb = 10;
double[] input = new double[nb];
double[] output = new double[nb];
for (int i = 0; i < nb; i++)
{
input[i] = Math.PI * rnd.NextDouble();
}
var softmax = new Softmax();
softmax.Activate(input, output);
Assert.AreEqual(input.ArgMax(), output.ArgMax());
double actualSum = output.Sum();
Assert.IsTrue(Math.Abs(1 - actualSum) <= 0.00001);
}
private Sequential CreateSequential(List <String> model)
{
// TODO just assumes it is all in a seq model the seq model should probably
// be in the JSON????
var seq = new Sequential(controller);
foreach (var l in model)
{
var config = JObject.Parse(l);
Layer layer = null;
switch ((string)config["name"])
{
case "linear":
layer = new Linear(controller, (int)config["input"], (int)config["output"]);
break;
case "softmax":
layer = new Softmax(controller, (int)config["dim"]);
break;
case "relu":
layer = new ReLU(controller);
break;
case "log":
layer = new Log(controller);
break;
case "dropout":
layer = new Dropout(controller, (float)config["rate"]);
break;
}
seq.AddLayer(layer);
}
return(seq);
}
public void TestRandomProbability()
{
var random = new Random(1337);
var tau = 200;
var softmax = new Softmax(tau, random);
var qValue = new QValue(new double[]
{
121, 231, 425, 676
});
var bestAction = PolicyHelpers.SelectMax(qValue, random);
var numSelected = new TestInstance[qValue.Count];
for (int i = 0; i < qValue.Count; i++)
{
numSelected[i] = new TestInstance()
{
Action = i
};
}
int numTests = 3000;
for (int i = 0; i < numTests; i++)
{
int action = softmax.Select(qValue);
numSelected[action].Count++;
}
numSelected = numSelected.OrderBy(x => x.Count).ToArray();
Assert.AreEqual(0, numSelected[0].Action);
Assert.AreEqual(1, numSelected[1].Action);
Assert.AreEqual(2, numSelected[2].Action);
Assert.AreEqual(3, numSelected[3].Action);
}
private async Task <Tuple <string, CommandInfo> > MostSimilarCommandAsync(string inputText, float threshhold = 0.95f)
{
if (string.IsNullOrWhiteSpace(inputText))
{
return(null);
}
if (threshhold < 0)
{
threshhold = 0;
}
if (threshhold > 1)
{
threshhold = 1;
}
var commandScores = new List <Tuple <string, double, CommandInfo> >();
foreach (var command in _commandService.Commands)
{
bool restrictedCommand = false;
foreach (var condition in command.Preconditions)
{
if (condition.Group == null && condition is RequireOwnerAttribute)
{
restrictedCommand = true;
break;
}
}
if (restrictedCommand)
{
continue;
}
double distance = LevenshteinDistance.Compute(command.Name, inputText);
commandScores.Add(Tuple.Create(command.Name, distance, command));
foreach (var alias in command.Aliases)
{
if (alias == command.Name)
{
continue;
}
double alias_distance = LevenshteinDistance.Compute(alias, inputText);
commandScores.Add(Tuple.Create(alias, distance, command));
}
}
var scores = commandScores.Select(x => x.Item2);
var probabilities_inverted = Softmax.Compute(scores);
var probabilities = probabilities_inverted.Select(x => 1 - x).ToList();
var commandProbabilities = new List <Tuple <string, double, CommandInfo> >();
if (commandScores.Count != probabilities.Count())
{
throw new ArgumentException(nameof(commandScores) + " length is not equal to " + nameof(probabilities));
}
for (int i = 0; i < commandScores.Count; ++i)
{
commandProbabilities.Add(Tuple.Create(commandScores[i].Item1, probabilities[i], commandScores[i].Item3));
}
commandProbabilities = commandProbabilities.Where(x => x.Item2 >= threshhold).OrderByDescending(x => x.Item2).ToList();
if (commandProbabilities.Count > 0)
{
var top = commandProbabilities.FirstOrDefault();
return(Tuple.Create(top.Item1, top.Item3));
}
else
{
return(null);
}
}
public Layer(int nCount, int index, ActivationSettings activationSettings)
{
NCount = nCount;
Index = index;
ActivationType = activationSettings.Type();
// Activation Setup
switch (activationSettings.Type())
{
case EActivationType.Invalid:
Activation = null;
throw new ArgumentException("Activation Type Invalid.");
case EActivationType.Arctan:
Activation = new Arctan();
break;
case EActivationType.BinaryStep:
Activation = new BinaryStep();
break;
case EActivationType.BipolarSigmoid:
Activation = new BipolarSigmoid();
break;
case EActivationType.ELU:
Activation = new ELU((ELUSettings)activationSettings);
break;
case EActivationType.HardSigmoid:
Activation = new HardSigmoid();
break;
case EActivationType.HardTanh:
Activation = new HardTanh();
break;
case EActivationType.Identity:
Activation = new Identity();
break;
case EActivationType.Logit:
Activation = new Logit();
break;
case EActivationType.LReLU:
Activation = new LReLU((LReLUSettings)activationSettings);
break;
case EActivationType.Mish:
Activation = new Mish();
break;
case EActivationType.ReLU:
Activation = new ReLU();
break;
case EActivationType.SeLU:
Activation = new SeLU();
break;
case EActivationType.Sigmoid:
Activation = new Sigmoid();
break;
case EActivationType.Softmax:
Activation = new Softmax();
break;
case EActivationType.Softplus:
Activation = new Softplus();
break;
case EActivationType.Softsign:
Activation = new Softsign();
break;
case EActivationType.Tanh:
Activation = new Tanh();
break;
default:
throw new ArgumentException("Activation Type Invalid.");
}
}
static void Main(string[] args)
{
int interation = 5000;
double eta = 0.25;
int neurons = 5;
var beta = 1;
var momentum = 0.9;
var inputs = new double[, ]
{
{ 0, 0 },
{ 1, 0 },
{ 0, 1 },
{ 1, 1 },
};
var xortargets = new double[]
{
0, 1, 1, 0
};
var ortargets = new double[]
{
0, 1, 1, 1
};
var andtargets = new double[]
{
0, 0, 0, 1
};
//=SIN(2*PI()*A10)+COS(4*PI()*A10)*RAND()*0.2
// where As are numbers from liner space between 0 and 1
var RandomInputs = new double[, ]
{
{ 0 },
{ 0.038461538 },
{ 0.076923077 },
{ 0.115384615 },
{ 0.153846154 },
{ 0.192307692 },
{ 0.230769231 },
{ 0.269230769 },
{ 0.307692308 },
{ 0.346153846 },
{ 0.384615385 },
{ 0.423076923 },
{ 0.461538462 },
{ 0.5 },
{ 0.538461538 },
{ 0.576923077 },
{ 0.615384615 },
{ 0.653846154 },
{ 0.692307692 },
{ 0.730769231 },
{ 0.769230769 },
{ 0.807692308 },
{ 0.846153846 },
{ 0.884615385 },
{ 0.923076923 },
{ 0.961538462 },
{ 1 }
};
var randomTargets = new double[]
{
0.006680055,
0.265486149,
0.483279114,
0.668364274,
0.763504244,
0.038461538,
0.798585083,
0.845324964,
0.793562162,
0.759179583,
0.686347672,
0.491122895,
0.243967266,
0.019711415,
-0.137190041,
-0.44675456,
-0.642939788,
-0.883257236,
-0.970691029,
-1.076569246,
-1.083657338,
-0.975380275,
-0.872895307,
-0.64233422,
-0.363677106,
-0.063115857,
0.186050921,
};
var randomTargets1 = (double[])randomTargets.Clone();
var logistic = new Logistic(beta);
var linear = new Linear(randomTargets1.Length);
var gateSoftmax = new Softmax(inputs.GetLength(0));
var gateLinear = new Linear(inputs.RowLength());
var xorPerc = new MultiLayerPerceptron(inputs, (double[])xortargets.Clone(), neurons, beta, momentum, gateLinear);
var orPerc = new MultiLayerPerceptron(inputs, (double[])ortargets.Clone(), neurons, beta, momentum, gateLinear);
var andPerc = new MultiLayerPerceptron(inputs, (double[])andtargets.Clone(), neurons, beta, momentum, gateLinear);
var randomPerc = new MultiLayerPerceptron(RandomInputs, randomTargets, neurons, beta, momentum, linear);
var tasks = new Task[]
{
Task.Factory.StartNew(() => xorPerc.Train(interation, eta)),
Task.Factory.StartNew(() => orPerc.Train(interation, eta)),
Task.Factory.StartNew(() => andPerc.Train(interation, eta)),
Task.Factory.StartNew(() => randomPerc.Train(interation + 40000, eta))
};
Console.Out.WriteLine("Start" + Environment.NewLine);
Task.WaitAll(tasks);
Console.Out.WriteLine("Stop" + Environment.NewLine);
xorPerc.Train(interation, eta);
Console.WriteLine("XOR");
xorPerc.ConfusionMatrix(inputs, xortargets);
Console.Out.WriteLine(Environment.NewLine);
Console.WriteLine("OR");
orPerc.ConfusionMatrix(inputs, ortargets);
Console.Out.WriteLine(Environment.NewLine);
Console.WriteLine("AND");
andPerc.ConfusionMatrix(inputs, andtargets);
Console.Out.WriteLine(Environment.NewLine);
Console.Out.WriteLine("MY random shit function from excel");
randomPerc.ConfusionMatrix(RandomInputs, randomTargets1);
Console.Out.WriteLine(Environment.NewLine);
Console.ReadKey();
}
private List <IKernelDescriptor> ReadDescriptors(JObject model)
{
List <IKernelDescriptor> dscps = model.SelectToken("descriptors").Select(layer => {
IKernelDescriptor descriptor = null;
String layerName = (String)layer.SelectToken("layer");
switch (layerName)
{
case "AvgPooling1D":
descriptor = new AvgPooling1D(
(int)layer.SelectToken("padding"),
(int)layer.SelectToken("stride"),
(int)layer.SelectToken("kernel_size"));
break;
case "GlobalAveragePooling1D":
descriptor = new GlobalAvgPooling1D();
break;
case "AvgPooling2D":
descriptor = new AvgPooling2D((int)layer.SelectToken("padding_vl"), (int)layer.SelectToken("padding_hz"),
(int)layer.SelectToken("stride_vl"), (int)layer.SelectToken("stride_hz"),
(int)layer.SelectToken("kernel_height"), (int)layer.SelectToken("kernel_width"));
break;
case "GlobalAveragePooling2D":
descriptor = new GlobalAvgPooling2D();
break;
case "BatchNormalization":
descriptor = new BatchNormalization(
(int)layer.SelectToken("epsilon"));
break;
case "Cropping1D":
descriptor = new Cropping1D(
(int)layer.SelectToken("trimBegin"),
(int)layer.SelectToken("trimEnd"));
break;
case "Cropping2D":
descriptor = new Cropping2D(
(int)layer.SelectToken("topTrim"),
(int)layer.SelectToken("bottomTrim"),
(int)layer.SelectToken("leftTrim"),
(int)layer.SelectToken("rightTrim"));
break;
case "MaxPooling1D":
descriptor = new MaxPooling1D(
(int)layer.SelectToken("padding"),
(int)layer.SelectToken("stride"),
(int)layer.SelectToken("kernel_size"));
break;
case "GlobalMaxPooling1D":
descriptor = new GlobalMaxPooling1D();
break;
case "MaxPooling2D":
descriptor = new MaxPooling2D((int)layer.SelectToken("padding_vl"), (int)layer.SelectToken("padding_hz"),
(int)layer.SelectToken("stride_vl"), (int)layer.SelectToken("stride_hz"),
(int)layer.SelectToken("kernel_height"), (int)layer.SelectToken("kernel_width"));
break;
case "GlobalMaxPooling2D":
descriptor = new GlobalMaxPooling2D();
break;
case "Convolution1D":
descriptor = new Convolution1D(
(int)layer.SelectToken("padding"),
(int)layer.SelectToken("stride"),
(int)layer.SelectToken("kernel_size"),
(int)layer.SelectToken("kernel_num"));
break;
case "Convolution2D":
descriptor = new Convolution2D((int)layer.SelectToken("padding_vl"), (int)layer.SelectToken("padding_hz"),
(int)layer.SelectToken("stride_vl"), (int)layer.SelectToken("stride_hz"),
(int)layer.SelectToken("kernel_height"), (int)layer.SelectToken("kernel_width"),
(int)layer.SelectToken("kernel_num"));
break;
case "Dense2D":
descriptor = new Dense2D((int)layer.SelectToken("units"));
break;
case "Input2D":
descriptor = new Input2D((int)layer.SelectToken("height"), (int)layer.SelectToken("width"),
(int)layer.SelectToken("channel"), (int)layer.SelectToken("batch"));
break;
case "Bias2D":
descriptor = new Bias2D();
break;
case "Permute":
descriptor = new Permute(
(int)layer.SelectToken("dim1"),
(int)layer.SelectToken("dim2"),
(int)layer.SelectToken("dim3"));
break;
case "Reshape":
descriptor = new Reshape2D(
(int)layer.SelectToken("height"),
(int)layer.SelectToken("width"),
(int)layer.SelectToken("channel"),
1);
break;
case "RepeatVector":
descriptor = new RepeatVector(
(int)layer.SelectToken("num"));
break;
case "SimpleRNN":
descriptor = new SimpleRNN(
(int)layer.SelectToken("units"),
(int)layer.SelectToken("input_dim"),
ANR((string)layer.SelectToken("activation")));
break;
case "LSTM":
descriptor = new LSTM(
(int)layer.SelectToken("units"),
(int)layer.SelectToken("input_dim"),
ANR((string)layer.SelectToken("activation")),
ANR((string)layer.SelectToken("rec_act")));
break;
case "GRU":
descriptor = new GRU(
(int)layer.SelectToken("units"),
(int)layer.SelectToken("input_dim"),
ANR((string)layer.SelectToken("activation")),
ANR((string)layer.SelectToken("rec_act")));
break;
case "ELu":
descriptor = new ELu(1);
break;
case "HardSigmoid":
descriptor = new HardSigmoid();
break;
case "ReLu":
descriptor = new ReLu();
break;
case "Sigmoid":
descriptor = new Sigmoid();
break;
case "Flatten":
descriptor = new Flatten();
break;
case "Softmax":
descriptor = new Softmax();
break;
case "SoftPlus":
descriptor = new SoftPlus();
break;
case "SoftSign":
descriptor = new Softsign();
break;
case "TanH":
descriptor = new TanH();
break;
default:
throw new Exception("Unknown layer type!");
}
return(descriptor);
}).ToList();
return(dscps);
}
public static void Run(bool verbose)
{
Stopwatch sw = new Stopwatch();
NdArray inputArrayCpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE));
NdArray inputArrayGpu = new NdArray(BenchDataMaker.GetRealArray(INPUT_SIZE));
Ensure.Argument(inputArrayGpu).NotNull();
Ensure.Argument(inputArrayCpu).NotNull();
//Linear
Linear linear = new Linear(verbose, INPUT_SIZE, OUTPUT_SIZE);
if (verbose)
{
RILogManager.Default?.EnterMethod(linear.Name);
}
sw.Restart();
NdArray[] gradArrayCpu = linear.Forward(verbose, inputArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
Ensure.Argument(gradArrayCpu).NotNull();
gradArrayCpu[0].Grad = gradArrayCpu[0].Data; // Use Data as Grad
sw.Restart();
linear.Backward(verbose, gradArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
if (linear.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = linear.Forward(verbose, inputArrayGpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
linear.Backward(verbose, gradArrayGpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
}
if (verbose)
{
RILogManager.Default?.ExitMethod(linear.Name);
}
//Tanh
Tanh tanh = new Tanh();
if (verbose)
{
RILogManager.Default?.EnterMethod(tanh.Name);
}
sw.Restart();
gradArrayCpu = tanh.Forward(verbose, inputArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
tanh.Backward(verbose, gradArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
if (tanh.SetGpuEnable(true))
{
HandleGPU(verbose, sw, tanh, inputArrayGpu);
}
if (verbose)
{
RILogManager.Default?.ExitMethod(tanh.Name);
}
//Sigmoid
Sigmoid sigmoid = new Sigmoid();
if (verbose)
{
RILogManager.Default?.EnterMethod(sigmoid.Name);
}
sw.Restart();
gradArrayCpu = sigmoid.Forward(verbose, inputArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
sigmoid.Backward(verbose, gradArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
if (sigmoid.SetGpuEnable(true))
{
HandleGPU(verbose, sw, sigmoid, inputArrayGpu);
}
if (verbose)
{
RILogManager.Default?.ExitMethod(tanh.Name);
}
//Softmax
Softmax sm = new Softmax();
RILogManager.Default?.EnterMethod(sm.Name);
sw.Restart();
gradArrayCpu = sm.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
sm.Backward(verbose, gradArrayCpu);
sw.Stop();
if (verbose)
{
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
if (verbose)
{
RILogManager.Default?.ExitMethod(sm.Name);
}
//Softplus
Softplus sp = new Softplus();
if (verbose)
{
RILogManager.Default?.EnterMethod(sp.Name);
}
sw.Restart();
gradArrayCpu = sp.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
sp.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
RILogManager.Default?.ExitMethod(sp.Name);
//ReLU
ReLU relu = new ReLU();
RILogManager.Default?.EnterMethod(relu.Name);
sw.Restart();
gradArrayCpu = relu.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
relu.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (relu.SetGpuEnable(true))
{
HandleGPU(verbose, sw, relu, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(relu.Name);
//LeakyReLU
LeakyReLU leakyRelu = new LeakyReLU();
RILogManager.Default?.EnterMethod(leakyRelu.Name);
sw.Restart();
gradArrayCpu = leakyRelu.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
leakyRelu.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (leakyRelu.SetGpuEnable(true))
{
HandleGPU(verbose, sw, leakyRelu, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(leakyRelu.Name);
//ReLuTanh
ReLuTanh rth = new ReLuTanh();
RILogManager.Default?.EnterMethod(rth.Name);
sw.Restart();
gradArrayCpu = rth.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
rth.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (rth.SetGpuEnable(true))
{
HandleGPU(verbose, sw, rth, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(rth.Name);
////Swish
//Swish swi = new Swish();
//RILogManager.Default?.SendDebug(swi.Name);
//sw.Restart();
//gradArrayCpu = swi.Forward(inputArrayCpu);
//sw.Stop();
//RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
//gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
//sw.Restart();
//swi.Backward(gradArrayCpu);
//sw.Stop();
//RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
NdArray inputImageArrayGpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5);
NdArray inputImageArrayCpu = new NdArray(BenchDataMaker.GetRealArray(3 * 256 * 256 * 5), new[] { 3, 256, 256 }, 5);
//MaxPooling
MaxPooling maxPooling = new MaxPooling(3);
RILogManager.Default?.EnterMethod(maxPooling.Name);
sw.Restart();
NdArray[] gradImageArrayCpu = maxPooling.Forward(verbose, inputImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
maxPooling.Backward(verbose, gradImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (maxPooling.SetGpuEnable(true))
{
sw.Restart();
maxPooling.Forward(verbose, inputImageArrayGpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
// There is no implementation for memory transfer only
RILogManager.Default?.SendDebug("Backward[Gpu] : None");
}
RILogManager.Default?.ExitMethod(maxPooling.Name);
//AvgPooling
AveragePooling avgPooling = new AveragePooling(3);
RILogManager.Default?.EnterMethod(avgPooling.Name);
sw.Restart();
gradImageArrayCpu = avgPooling.Forward(verbose, inputImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
avgPooling.Backward(verbose, gradImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
RILogManager.Default?.ExitMethod(avgPooling.Name);
//Conv2D
Convolution2D conv2d = new Convolution2D(verbose, 3, 3, 3);
RILogManager.Default?.EnterMethod(conv2d.Name);
sw.Restart();
gradImageArrayCpu = conv2d.Forward(verbose, inputImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
conv2d.Backward(verbose, gradImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (conv2d.SetGpuEnable(true))
{
HandleGPU(verbose, sw, conv2d, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(conv2d.Name);
//Deconv2D
Deconvolution2D deconv2d = new Deconvolution2D(verbose, 3, 3, 3);
RILogManager.Default?.EnterMethod(deconv2d.Name);
sw.Restart();
gradImageArrayCpu = deconv2d.Forward(verbose, inputImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradImageArrayCpu[0].Grad = gradImageArrayCpu[0].Data;
sw.Restart();
deconv2d.Backward(verbose, gradImageArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (deconv2d.SetGpuEnable(true))
{
HandleGPU(verbose, sw, deconv2d, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(deconv2d.Name);
//Dropout
Dropout dropout = new Dropout();
RILogManager.Default?.EnterMethod(dropout.Name);
sw.Restart();
gradArrayCpu = dropout.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
dropout.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (dropout.SetGpuEnable(true))
{
sw.Restart();
NdArray[] gradArrayGpu = dropout.Forward(verbose, inputArrayGpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayGpu[0].Grad = gradArrayGpu[0].Data;
sw.Restart();
dropout.Backward(verbose, gradArrayGpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Gpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
}
RILogManager.Default?.ExitMethod(dropout.Name);
//ArcSinH
ArcSinH a = new ArcSinH();
RILogManager.Default?.EnterMethod(a.Name);
sw.Restart();
gradArrayCpu = a.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
a.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (a.SetGpuEnable(true))
{
HandleGPU(verbose, sw, a, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(a.Name);
//ELU
ELU e = new ELU();
RILogManager.Default?.EnterMethod(e.Name);
sw.Restart();
gradArrayCpu = e.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
e.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
RILogManager.Default?.ExitMethod(e.Name);
//LeakyReluShifted
LeakyReLUShifted lrs = new LeakyReLUShifted();
RILogManager.Default?.EnterMethod(lrs.Name);
sw.Restart();
gradArrayCpu = lrs.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
lrs.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (lrs.SetGpuEnable(true))
{
HandleGPU(verbose, sw, lrs, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(lrs.Name);
//Logistic
LogisticFunction lf = new LogisticFunction();
RILogManager.Default?.EnterMethod(lf.Name);
sw.Restart();
gradArrayCpu = lf.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
lf.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (lf.SetGpuEnable(true))
{
HandleGPU(verbose, sw, lf, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(lf.Name);
//MaxMinusOne
MaxMinusOne mmo = new MaxMinusOne();
RILogManager.Default?.EnterMethod(mmo.Name);
sw.Restart();
gradArrayCpu = mmo.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
mmo.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (mmo.SetGpuEnable(true))
{
HandleGPU(verbose, sw, mmo, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(mmo.Name);
//ScaledELU
ScaledELU se = new ScaledELU();
RILogManager.Default?.EnterMethod(se.Name);
sw.Restart();
gradArrayCpu = se.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
se.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (se.SetGpuEnable(true))
{
HandleGPU(verbose, sw, se, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(se.Name);
//Sine
Sine s = new Sine();
RILogManager.Default?.EnterMethod(s.Name);
sw.Restart();
gradArrayCpu = s.Forward(verbose, inputArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Forward [Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
gradArrayCpu[0].Grad = gradArrayCpu[0].Data;
sw.Restart();
s.Backward(verbose, gradArrayCpu);
sw.Stop();
RILogManager.Default?.SendDebug("Backward[Cpu] : " + (sw.ElapsedTicks / (Stopwatch.Frequency / (1000L * 1000L))).ToString("n0") + "μｓ");
if (s.SetGpuEnable(true))
{
HandleGPU(verbose, sw, s, inputArrayGpu);
}
RILogManager.Default?.ExitMethod(s.Name);
}
public AttentionCore(int hiddens, int sequences)
{
this.hiddens = hiddens;
this.sequences = sequences;
this.softmax = new Softmax(this.sequences);
}
public Attention(Variable <T> encoderHiddenStates, Variable <T> decoderHiddenStates, long attentionDim)
{
AttentionDim = attentionDim;
EncoderHiddenStates = encoderHiddenStates;
DecoderHiddenStates = decoderHiddenStates;
// one goal is, try to make batchSize and encoderSeqLength unknown at symbol layer
// so, in LSTM outer op, we can create one graph and one sub-executor, and applied for
// different encoderSeqLength and batchSize.
Util.EnsureEqual(3, EncoderHiddenStates.Shape.Rank, "states layout: (encoderSeqLength, batch, encoderHiddenSize)");
Util.EnsureTrue(EncoderHiddenStates.Shape[2] > 0, "states should be determined.");
EncoderHiddenSize = EncoderHiddenStates.Shape[2];
Util.EnsureEqual(2, DecoderHiddenStates.Shape.Rank, "DecoderHiddenStates layout: (batch, decoderHiddenSize)");
Util.EnsureTrue(DecoderHiddenStates.Shape[1] > 0, "DecoderHiddenStates should be determined.");
DecoderHiddenSize = DecoderHiddenStates.Shape[1];
var scaleWh = Sqrt(12.0.AsScalar <T>() / ((double)(AttentionDim + EncoderHiddenSize)).AsScalar <T>());
Wh = Parameter(scaleWh * (RandomUniform <T>(Shape.Create(EncoderHiddenSize, AttentionDim), 0UL, 0UL) - 0.5.AsScalar <T>()));
var scaleWd = Sqrt(12.0.AsScalar <T>() / ((double)(AttentionDim + DecoderHiddenSize)).AsScalar <T>());
Wd = Parameter(scaleWd * (RandomUniform <T>(Shape.Create(DecoderHiddenSize, AttentionDim), 0UL, 0UL) - 0.5.AsScalar <T>()));
var scaleV = Sqrt(12.0.AsScalar <T>() / ((double)(AttentionDim)).AsScalar <T>());
V = Parameter(scaleV * (RandomUniform <T>(Shape.Create(AttentionDim, 1), 0UL, 0UL) - 0.5.AsScalar <T>()));
// build the graph
var h = EncoderHiddenStates; // (n*b,He) // He denotes hiddenSize of encoder
var d = DecoderHiddenStates; // (b,Hd) // Hd denotes hiddenSize of decoder
var whh = Dot(h.Reshape(-1, EncoderHiddenSize), Wh); // shape (n*b,K) K denotes attentionDim
var wdd = Dot(d, Wd); // shape (b,K)
// to add whh and wdd, we need broadcast, for this, we need to know at least n or b.
// The decision here is to make b known at symbolic layer, because then you can have
// flexibility on n (EncoderSeqLength), easier for making bucket.
// another issue is, our backward of add has some issue dealing with 3d array which has broadcast
// so, we can reshape them into 2d tensor here:
// initial shape: (n*b,K) + (b,K)
// reshape for the boadcast: (n,b*K) + (b*K) (for broadcasting, (b*K) will broadcast to (1,b*K)
// then: (n,b*K) + (b*K) = (n,b*K)
// reshape result to (n*b,K)
BatchSize = EncoderHiddenStates.Shape[1];
Util.EnsureTrue(BatchSize > 0, "Batch need to be determined.");
Util.EnsureTrue(BatchSize == DecoderHiddenStates.Shape[0]);
var sum = (whh.Reshape(-1, BatchSize * AttentionDim) + wdd.Reshape(-1)).Reshape(-1, AttentionDim);
// tanh, shape no change (n*b,K)
var whd = new ActivationTanh <T>(sum);
// (n*b,K) dot (K,1) = (n*b,1) => reshape to (n,b)
var u = Dot(whd.Output, V).Reshape(-1, BatchSize);
// same shape (n,b)
var softmax = new Softmax <T>(u);
// sum (n,b) * (n,b,d)
var reduce = new AttentionReduce <T>(softmax.Output.Reshape(-1, BatchSize), h);
Output = reduce.Output;
}
public void ProcessMessage(string json_message, MonoBehaviour owner, Action <string> response)
{
Command msgObj = JsonUtility.FromJson <Command> (json_message);
try
{
switch (msgObj.objectType)
{
case "Optimizer":
{
if (msgObj.functionCall == "create")
{
string optimizer_type = msgObj.tensorIndexParams[0];
// Extract parameters
List <int> p = new List <int>();
for (int i = 1; i < msgObj.tensorIndexParams.Length; i++)
{
p.Add(int.Parse(msgObj.tensorIndexParams[i]));
}
List <float> hp = new List <float>();
for (int i = 0; i < msgObj.hyperParams.Length; i++)
{
hp.Add(float.Parse(msgObj.hyperParams[i]));
}
Optimizer optim = null;
if (optimizer_type == "sgd")
{
optim = new SGD(this, p, hp[0], hp[1], hp[2]);
}
else if (optimizer_type == "rmsprop")
{
optim = new RMSProp(this, p, hp[0], hp[1], hp[2], hp[3]);
}
else if (optimizer_type == "adam")
{
optim = new Adam(this, p, hp[0], hp[1], hp[2], hp[3], hp[4]);
}
response(optim.Id.ToString());
return;
}
else
{
Optimizer optim = this.GetOptimizer(msgObj.objectIndex);
response(optim.ProcessMessage(msgObj, this));
return;
}
}
case "FloatTensor":
{
if (msgObj.objectIndex == 0 && msgObj.functionCall == "create")
{
FloatTensor tensor = floatTensorFactory.Create(_shape: msgObj.shape, _data: msgObj.data, _shader: this.Shader);
response(tensor.Id.ToString());
return;
}
else
{
FloatTensor tensor = floatTensorFactory.Get(msgObj.objectIndex);
// Process message's function
response(tensor.ProcessMessage(msgObj, this));
return;
}
}
case "IntTensor":
{
if (msgObj.objectIndex == 0 && msgObj.functionCall == "create")
{
int[] data = new int[msgObj.data.Length];
for (int i = 0; i < msgObj.data.Length; i++)
{
data[i] = (int)msgObj.data[i];
}
IntTensor tensor = intTensorFactory.Create(_shape: msgObj.shape, _data: data);
response(tensor.Id.ToString());
return;
}
else
{
IntTensor tensor = intTensorFactory.Get(msgObj.objectIndex);
// Process message's function
response(tensor.ProcessMessage(msgObj, this));
return;
}
}
case "agent":
{
if (msgObj.functionCall == "create")
{
Layer model = (Layer)GetModel(int.Parse(msgObj.tensorIndexParams[0]));
Optimizer optimizer = optimizers[int.Parse(msgObj.tensorIndexParams[1])];
response(new Syft.NN.RL.Agent(this, model, optimizer).Id.ToString());
return;
}
//Debug.Log("Getting Model:" + msgObj.objectIndex);
Syft.NN.RL.Agent agent = this.GetAgent(msgObj.objectIndex);
response(agent.ProcessMessageLocal(msgObj, this));
return;
}
case "model":
{
if (msgObj.functionCall == "create")
{
string model_type = msgObj.tensorIndexParams[0];
Debug.LogFormat("<color=magenta>createModel:</color> {0}", model_type);
if (model_type == "linear")
{
response(this.BuildLinear(msgObj.tensorIndexParams).Id.ToString());
return;
}
else if (model_type == "relu")
{
response(this.BuildReLU().Id.ToString());
return;
}
else if (model_type == "log")
{
response(this.BuildLog().Id.ToString());
return;
}
else if (model_type == "dropout")
{
response(this.BuildDropout(msgObj.tensorIndexParams).Id.ToString());
return;
}
else if (model_type == "sigmoid")
{
response(this.BuildSigmoid().Id.ToString());
return;
}
else if (model_type == "sequential")
{
response(this.BuildSequential().Id.ToString());
return;
}
else if (model_type == "softmax")
{
response(this.BuildSoftmax(msgObj.tensorIndexParams).Id.ToString());
return;
}
else if (model_type == "logsoftmax")
{
response(this.BuildLogSoftmax(msgObj.tensorIndexParams).Id.ToString());
return;
}
else if (model_type == "tanh")
{
response(new Tanh(this).Id.ToString());
return;
}
else if (model_type == "crossentropyloss")
{
response(new CrossEntropyLoss(this, int.Parse(msgObj.tensorIndexParams[1])).Id.ToString());
return;
}
else if (model_type == "categorical_crossentropy")
{
response(new CategoricalCrossEntropyLoss(this).Id.ToString());
return;
}
else if (model_type == "nllloss")
{
response(new NLLLoss(this).Id.ToString());
return;
}
else if (model_type == "mseloss")
{
response(new MSELoss(this).Id.ToString());
return;
}
else if (model_type == "embedding")
{
response(new Embedding(this, int.Parse(msgObj.tensorIndexParams[1]), int.Parse(msgObj.tensorIndexParams[2])).Id.ToString());
return;
}
else
{
Debug.LogFormat("<color=red>Model Type Not Found:</color> {0}", model_type);
}
}
else
{
//Debug.Log("Getting Model:" + msgObj.objectIndex);
Model model = this.GetModel(msgObj.objectIndex);
response(model.ProcessMessage(msgObj, this));
return;
}
response("Unity Error: SyftController.processMessage: Command not found:" + msgObj.objectType + ":" + msgObj.functionCall);
return;
}
case "controller":
{
if (msgObj.functionCall == "num_tensors")
{
response(floatTensorFactory.Count() + "");
return;
}
else if (msgObj.functionCall == "num_models")
{
response(models.Count + "");
return;
}
else if (msgObj.functionCall == "new_tensors_allowed")
{
Debug.LogFormat("New Tensors Allowed:{0}", msgObj.tensorIndexParams[0]);
if (msgObj.tensorIndexParams[0] == "True")
{
allow_new_tensors = true;
}
else if (msgObj.tensorIndexParams[0] == "False")
{
allow_new_tensors = false;
}
else
{
throw new Exception("Invalid parameter for new_tensors_allowed. Did you mean true or false?");
}
response(allow_new_tensors + "");
return;
}
else if (msgObj.functionCall == "load_floattensor")
{
FloatTensor tensor = floatTensorFactory.Create(filepath: msgObj.tensorIndexParams[0], _shader: this.Shader);
response(tensor.Id.ToString());
return;
}
else if (msgObj.functionCall == "set_seed")
{
Random.InitState(int.Parse(msgObj.tensorIndexParams[0]));
response("Random seed set!");
return;
}
else if (msgObj.functionCall == "concatenate")
{
List <int> tensor_ids = new List <int>();
for (int i = 1; i < msgObj.tensorIndexParams.Length; i++)
{
tensor_ids.Add(int.Parse(msgObj.tensorIndexParams[i]));
}
FloatTensor result = Functional.Concatenate(floatTensorFactory, tensor_ids, int.Parse(msgObj.tensorIndexParams[0]));
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "ones")
{
int[] dims = new int[msgObj.tensorIndexParams.Length];
for (int i = 0; i < msgObj.tensorIndexParams.Length; i++)
{
dims[i] = int.Parse(msgObj.tensorIndexParams[i]);
}
FloatTensor result = Functional.Ones(floatTensorFactory, dims);
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "randn")
{
int[] dims = new int[msgObj.tensorIndexParams.Length];
for (int i = 0; i < msgObj.tensorIndexParams.Length; i++)
{
dims[i] = int.Parse(msgObj.tensorIndexParams[i]);
}
FloatTensor result = Functional.Randn(floatTensorFactory, dims);
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "random")
{
int[] dims = new int[msgObj.tensorIndexParams.Length];
for (int i = 0; i < msgObj.tensorIndexParams.Length; i++)
{
dims[i] = int.Parse(msgObj.tensorIndexParams[i]);
}
FloatTensor result = Functional.Random(floatTensorFactory, dims);
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "zeros")
{
int[] dims = new int[msgObj.tensorIndexParams.Length];
for (int i = 0; i < msgObj.tensorIndexParams.Length; i++)
{
dims[i] = int.Parse(msgObj.tensorIndexParams[i]);
}
FloatTensor result = Functional.Zeros(floatTensorFactory, dims);
response(result.Id.ToString());
return;
}
else if (msgObj.functionCall == "model_from_json")
{
Debug.Log("Loading Model from JSON:");
var json_str = msgObj.tensorIndexParams[0];
var config = JObject.Parse(json_str);
Sequential model;
if ((string)config["class_name"] == "Sequential")
{
model = this.BuildSequential();
}
else
{
response("Unity Error: SyftController.processMessage: while Loading model, Class :" + config["class_name"] + " is not implemented");
return;
}
for (int i = 0; i < config["config"].ToList().Count; i++)
{
var layer_desc = config["config"][i];
var layer_config_desc = layer_desc["config"];
if ((string)layer_desc["class_name"] == "Linear")
{
int previous_output_dim;
if (i == 0)
{
previous_output_dim = (int)layer_config_desc["batch_input_shape"][layer_config_desc["batch_input_shape"].ToList().Count - 1];
}
else
{
previous_output_dim = (int)layer_config_desc["units"];
}
string[] parameters = { "linear", previous_output_dim.ToString(), layer_config_desc["units"].ToString(), "Xavier" };
Layer layer = this.BuildLinear(parameters);
model.AddLayer(layer);
string activation_name = layer_config_desc["activation"].ToString();
if (activation_name != "linear")
{
Layer activation;
if (activation_name == "softmax")
{
parameters = new string[] { activation_name, "1" };
activation = this.BuildSoftmax(parameters);
}
else if (activation_name == "relu")
{
activation = this.BuildReLU();
}
else
{
response("Unity Error: SyftController.processMessage: while Loading activations, Activation :" + activation_name + " is not implemented");
return;
}
model.AddLayer(activation);
}
}
else
{
response("Unity Error: SyftController.processMessage: while Loading layers, Layer :" + layer_desc["class_name"] + " is not implemented");
return;
}
}
response(model.Id.ToString());
return;
}
else if (msgObj.functionCall == "from_proto")
{
Debug.Log("Loading Model from ONNX:");
var filename = msgObj.tensorIndexParams[0];
var input = File.OpenRead(filename);
ModelProto modelProto = ModelProto.Parser.ParseFrom(input);
Sequential model = this.BuildSequential();
foreach (NodeProto node in modelProto.Graph.Node)
{
Layer layer;
GraphProto g = ONNXTools.GetSubGraphFromNodeAndMainGraph(node, modelProto.Graph);
if (node.OpType == "Gemm")
{
layer = new Linear(this, g);
}
else if (node.OpType == "Dropout")
{
layer = new Dropout(this, g);
}
else if (node.OpType == "Relu")
{
layer = new ReLU(this, g);
}
else if (node.OpType == "Softmax")
{
layer = new Softmax(this, g);
}
else
{
response("Unity Error: SyftController.processMessage: Layer not yet implemented for deserialization:");
return;
}
model.AddLayer(layer);
}
response(model.Id.ToString());
return;
}
else if (msgObj.functionCall == "to_proto")
{
ModelProto model = this.ToProto(msgObj.tensorIndexParams);
string filename = msgObj.tensorIndexParams[2];
string type = msgObj.tensorIndexParams[3];
if (type == "json")
{
response(model.ToString());
}
else
{
using (var output = File.Create(filename))
{
model.WriteTo(output);
}
response(new FileInfo(filename).FullName);
}
return;
}
response("Unity Error: SyftController.processMessage: Command not found:" + msgObj.objectType + ":" + msgObj.functionCall);
return;
}
case "Grid":
if (msgObj.functionCall == "learn")
{
var inputId = int.Parse(msgObj.tensorIndexParams[0]);
var targetId = int.Parse(msgObj.tensorIndexParams[1]);
response(this.grid.Run(inputId, targetId, msgObj.configurations, owner));
return;
}
if (msgObj.functionCall == "getResults")
{
this.grid.GetResults(msgObj.experimentId, response);
return;
}
// like getResults but doesn't pause to wait for results
// this function will return right away telling you if
// it knows whether or not it is done
if (msgObj.functionCall == "checkStatus")
{
this.grid.CheckStatus(msgObj.experimentId, response);
return;
}
break;
default:
break;
}
}
catch (Exception e)
{
Debug.LogFormat("<color=red>{0}</color>", e.ToString());
response("Unity Error: " + e.ToString());
return;
}
// If not executing createTensor or tensor function, return default error.
response("Unity Error: SyftController.processMessage: Command not found:" + msgObj.objectType + ":" + msgObj.functionCall);
return;
}
