public static Pixbuf Get(ManagedCNN cnn, int layer) { if (layer >= 0 && layer < cnn.Layers.Count && cnn.Layers[layer].Type == LayerTypes.Convolution) { var Transposed = new ManagedArray(cnn.Layers[layer].Bias); ManagedMatrix.Transpose(Transposed, cnn.Layers[layer].Bias); var pixbuf = new Pixbuf(Colorspace.Rgb, false, 8, Transposed.x, Transposed.y); // Get normalization values double min = Double.MaxValue; double max = Double.MinValue; FullyConnected.GetNormalization(Transposed, ref min, ref max); Activation.Draw(pixbuf, Transposed, min, max); ManagedOps.Free(Transposed); return(pixbuf); } // return empty pixbuf return(new Pixbuf(Colorspace.Rgb, false, 8, 1, 1)); }
public static Pixbuf Get(ManagedCNN cnn, int layer, int i, int j) { if (layer >= 0 && layer < cnn.Layers.Count && cnn.Layers[layer].Type == LayerTypes.Convolution && i >= 0 && i < cnn.Layers[layer].FeatureMap.i && j >= 0 && j < cnn.Layers[layer].FeatureMap.j) { var FeatureMap = new ManagedArray(cnn.Layers[layer].FeatureMap.x, cnn.Layers[layer].FeatureMap.y, cnn.Layers[layer].FeatureMap.z); var Transposed = new ManagedArray(FeatureMap); var pixbuf = new Pixbuf(Colorspace.Rgb, false, 8, FeatureMap.y, FeatureMap.x); ManagedOps.Copy4DIJ2D(FeatureMap, cnn.Layers[layer].FeatureMap, i, j); ManagedMatrix.Transpose(Transposed, FeatureMap); // Get normalization values double min = Double.MaxValue; double max = Double.MinValue; FullyConnected.GetNormalization(Transposed, ref min, ref max); Activation.Draw(pixbuf, Transposed, min, max); ManagedOps.Free(Transposed); return(pixbuf); } // return empty pixbuf return(new Pixbuf(Colorspace.Rgb, false, 8, 1, 1)); }
private Layer ConvertFullyConnected(tflite.Operator op) { var inputs = op.GetInputsArray(); var input = _graph.Tensors(inputs[0]).Value; var options = op.BuiltinOptions <tflite.FullyConnectedOptions>().Value; var weights = _graph.Tensors(inputs[1]).Value; var bias = _graph.Tensors(inputs[2]).Value; if (input.ShapeLength == 4 && (input.Shape(1) != 1 || input.Shape(2) != 1)) { var flatten = new TensorflowFlatten(input.GetShapeArray().ToNCHW()); var layer = new FullyConnected(flatten.Output.Dimensions, _model.GetTensor <float>(weights), _model.GetTensor <float>(bias), options.FusedActivationFunction.ToActivationFunction()); layer.Input.SetConnection(flatten.Output); _inputs.Add(flatten.Input, inputs[0]); _outputs.Add(op.Outputs(0), layer.Output); return(layer); } else { var layer = new FullyConnected(input.GetShapeArray().ToNCHW(), _model.GetTensor <float>(weights), _model.GetTensor <float>(bias), options.FusedActivationFunction.ToActivationFunction()); _inputs.Add(layer.Input, inputs[0]); _outputs.Add(op.Outputs(0), layer.Output); return(layer); } }
public static Model ConvolutionalNeuralNetworkModel() { var images = Variable <float>(); var labels = Variable <float>(); ILayer <float> net = new Reshape <float>(images, PartialShape.Create(-1, 1, 28, 28)); net = new Convolution2D <float>(net.Output, 5, 5, 16); net = new ActivationReLU <float>(net.Output); net = new Pooling2D <float>(net.Output, PoolingMode.MAX, 2, 2, 2, 2); net = new Convolution2D <float>(net.Output, 5, 5, 32); net = new ActivationTanh <float>(net.Output); net = new Pooling2D <float>(net.Output, PoolingMode.MAX, 2, 2, 2, 2); net = new Reshape <float>(net.Output, PartialShape.Create(-1, net.Output.Shape.Skip(1).Aggregate(ScalarOps.Mul))); net = new FullyConnected <float>(net.Output, 50); net = new ActivationTanh <float>(net.Output); net = new FullyConnected <float>(net.Output, 10); return(new Model { Loss = new SoftmaxCrossEntropy <float>(net.Output, labels), Images = images, Labels = labels }); }
public void Infer(FullyConnected layer, FullyConnectedLayerArgument argument, InferenceContext context) { var inputAlloc = context.MainMemoryMap[layer.Input.Connection.From]; var outputAlloc = context.MainMemoryMap[layer.Output]; argument.MainMemoryInputAddress = inputAlloc.GetAddress(); argument.MainMemoryOutputAddress = outputAlloc.GetAddress(); }
public FullyConnectedLayerArgument Convert(FullyConnected layer, ConvertContext context) { return new FullyConnectedLayerArgument { InputChannels = (uint)layer.Input.Dimensions[1], OutputChannels = (uint)layer.Output.Dimensions[1], Activation = layer.FusedActivationFunction, Weights = layer.Weights.ToArray(), Bias = layer.Bias.ToArray() }; }
private Layer ConvertMul(paddle.OpDesc op) { var x = GetParameter(op.Inputs, "X").Arguments[0]; var y = GetParameter(op.Inputs, "Y").Arguments[0]; var output = GetParameter(op.Outputs, "Out").Arguments[0]; var layer = new FullyConnected(GetVarShape(x), LoadVarData <float>(y).Transpose(new[] { 1, 0 }), null, ActivationFunctionType.Linear); _inputs.Add(layer.Input, x); _outputs.Add(output, layer.Output); return(layer); }
public static Symbol CreateLenet() { Symbol data = Symbol.Variable("data"); Symbol data_label = Symbol.Variable("data_label"); // first conv // mx.symbol.Convolution(data = data, kernel = (5, 5), num_filter = 20) Symbol conv1 = new Convolution(new Shape(5, 5), 20).CreateSymbol(data); // tanh1 = mx.symbol.Activation(data=conv1, act_type="tanh") Symbol tanh1 = new Activation().CreateSymbol(conv1); // pool1 = mx.symbol.Pooling(data=tanh1, pool_type="max", kernel = (2,2), stride = (2,2)) Symbol pool1 = new Pooling(new Shape(2, 2), new Shape(2, 2)).CreateSymbol(tanh1); // second conv // conv2 = mx.symbol.Convolution(data=pool1, kernel=(5,5), num_filter=50) Symbol conv2 = new Convolution(new Shape(5, 5), 50).CreateSymbol(pool1); // tanh2 = mx.symbol.Activation(data=conv2, act_type="tanh") Symbol tanh2 = new Activation().CreateSymbol(conv2); // pool2 = mx.symbol.Pooling(data=tanh2, pool_type="max", kernel = (2,2), stride = (2,2)) Symbol pool2 = new Pooling(new Shape(2, 2), new Shape(2, 2)).CreateSymbol(tanh2); // first fullc // flatten = mx.symbol.Flatten(data=pool2) Symbol flatten = new Flatten().CreateSymbol(pool2); // fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=500) Symbol fc1 = new FullyConnected(500).CreateSymbol(flatten); // tanh3 = mx.symbol.Activation(data=fc1, act_type="tanh") Symbol tanh3 = new Activation().CreateSymbol(fc1); // second fullc // fc2 = mx.symbol.FullyConnected(data=tanh3, num_hidden=num_classes) Symbol fc2 = new FullyConnected(10).CreateSymbol(tanh3); // loss // lenet = mx.symbol.SoftmaxOutput(data=fc2, name='softmax') Symbol lenet = new SoftmaxOutput().CreateSymbol(fc2, data_label); System.IO.File.WriteAllText("lenet.json", lenet.ToJSON()); foreach (var item in lenet.ListAuxiliaryStates()) { Console.WriteLine(item); } return(lenet); }
public static Model MultiLayerPerceptronModel() { var images = Variable <float>(PartialShape.Create(-1, 28 * 28)); ILayer <float> net = new FullyConnected <float>(images, 128); net = new ActivationReLU <float>(net.Output); net = new FullyConnected <float>(net.Output, 64); net = new ActivationReLU <float>(net.Output); net = new FullyConnected <float>(net.Output, 10); var labels = Variable <float>(PartialShape.Create(-1, 10)); return(new Model { Loss = new SoftmaxCrossEntropy <float>(net.Output, labels), Images = images, Labels = labels }); }
static void Main(string[] args) { Operations K = new Operations(); //Load array to the tensor NDArray x = new NDArray(3, 3); x.Load(2, 4, 6, 1, 3, 5, 2, 3, 5); x.Print("Load X Values"); NDArray y = new NDArray(3, 1); y.Load(1, 0, 1); y.Print("Load Y Values"); //Create two layers, one with 6 neurons and another with 1 FullyConnected fc1 = new FullyConnected(3, 6, "relu"); FullyConnected fc2 = new FullyConnected(6, 1, "sigmoid"); //Connect input by passing data from one layer to another fc1.Forward(x); fc2.Forward(fc1.Output); var preds = fc2.Output; preds.Print("Predictions"); //Calculate the mean square error cost between the predicted and expected values BaseCost cost = new BinaryCrossEntropy(); var costValues = cost.Forward(preds, y); costValues.Print("BCE Cost"); //Calculate the mean absolute metric value for the predicted vs expected values BaseMetric metric = new BinaryAccuacy(); var metricValues = metric.Calculate(preds, y); metricValues.Print("Acc Metric"); var grad = cost.Backward(preds, y); fc2.Backward(grad); fc1.Backward(fc2.InputGrad); fc1.PrintParams(); Console.ReadLine(); }
private void DrawFullyConnectedLayers() { var featurevector = FullyConnected.Get(cnn.FeatureVector); var output = FullyConnected.Get(cnn.Output); var weights = FullyConnected.Get(cnn.Weights, false); var bias = FullyConnected.Get(cnn.Bias, false); if (IsActivated) { RenderScaled(FeatureVector, featurevector); RenderScaled(Output, output); RenderScaled(Weights, weights); RenderScaled(NetworkBias, bias); } Throw(featurevector, output, weights, bias); }
static void Main(string[] args) { Operations K = new Operations(); //Load array to the tensor NDArray x = new NDArray(3, 3); x.Load(2, 4, 6, 1, 3, 5, 2, 3, 5); x.Print("Load X Values"); NDArray y = new NDArray(3, 1); y.Load(20, 15, 15); y.Print("Load Y Values"); //Create two layers, one with 6 neurons and another with 1 FullyConnected fc1 = new FullyConnected(3, 6, "relu"); FullyConnected fc2 = new FullyConnected(6, 1, "relu"); //Connect input by passing data from one layer to another fc1.Forward(x); fc2.Forward(fc1.Output); var preds = fc2.Output; preds.Print("Predictions"); //Calculate the mean square error cost between the predicted and expected values BaseCost cost = new MeanSquaredError(); var costValues = cost.Forward(preds, y); costValues.Print("MSE Cost"); //Calculate the mean absolute metric value for the predicted vs expected values BaseMetric metric = new MeanAbsoluteError(); var metricValues = metric.Calculate(preds, y); metricValues.Print("MAE Metric"); Console.ReadLine(); }
public Decoder(int sequenceLength, int vocabularySize, int wordVectorSize, int hiddenSize) : base(sequenceLength, sequenceLength * vocabularySize) { this.embedding = new Embedding(sequenceLength, vocabularySize, wordVectorSize, (fanIn, fanOut) => 0.01 * Initializers.LeCunNormal(fanIn)); this.recurrent = new LSTM(wordVectorSize, hiddenSize, sequenceLength, true, false, (fanIn, fanOut) => Initializers.LeCunNormal(fanIn)); this.attention = new Attention(hiddenSize, sequenceLength); this.fullyConnected = new FullyConnected(hiddenSize * 2, sequenceLength, sequenceLength * vocabularySize, (fanIn, fanOut) => Initializers.LeCunNormal(fanIn)); this.weights = new double[this.embedding.Weights.Length + this.recurrent.Weights.Length + this.fullyConnected.Weights.Length]; for (int i = 0; i < this.embedding.Weights.Length; i++) { this.weights[i] = this.embedding.Weights[i]; } for (int i = 0, j = this.embedding.Weights.Length; i < this.recurrent.Weights.Length; i++, j++) { this.weights[j] = this.recurrent.Weights[i]; } for (int i = 0, j = this.embedding.Weights.Length + this.recurrent.Weights.Length; i < this.fullyConnected.Weights.Length; i++, j++) { this.weights[j] = this.fullyConnected.Weights[i]; } }
static void Main(string[] args) { //Load array to the tensor NDArray x = new NDArray(1, 3); x.Load(1, 2, 3); x.Print("Load array"); //Create two layers, one with 6 neurons and another with 1 FullyConnected fc1 = new FullyConnected(3, 6, "relu"); FullyConnected fc2 = new FullyConnected(6, 1, "sigmoid"); //Connect input by passing data from one layer to another fc1.Forward(x); x = fc1.Output; x.Print("FC1 Output"); fc2.Forward(x); x = fc2.Output; x.Print("FC2 Output"); Console.ReadLine(); }
private Layer ConvertInnerProduct(LayerParameter layerParam) { var input = _outputs[layerParam.Bottom[0]]; var param = layerParam.InnerProductParam; var weights = LoadBlob(layerParam.Blobs[0]); if (input.Dimensions.Length == 4 && (input.Dimensions[2] != 1 || input.Dimensions[3] != 1)) { var flatten = new Reshape(input.Dimensions, new[] { -1, input.Dimensions.GetSize() }); var layer = new FullyConnected(flatten.Output.Dimensions, weights, null, ActivationFunctionType.Linear); flatten.Input.SetConnection(input); layer.Input.SetConnection(flatten.Output); _outputs[layerParam.Top[0]] = layer.Output; return(layer); } else { var layer = new FullyConnected(input.Dimensions, weights, null, ActivationFunctionType.Linear); layer.Input.SetConnection(input); _outputs[layerParam.Top[0]] = layer.Output; return(layer); } }
public Model(Context ctx, Config cfg, bool isTraining = true, bool usingCuDnn = true) { Config = cfg; IsTraining = isTraining; UsingCuDnn = usingCuDnn; Inputs = Variable <int>(PartialShape.Create(cfg.NumSteps, cfg.BatchSize)); Targets = Variable <int>(PartialShape.Create(cfg.NumSteps, cfg.BatchSize)); // embedding Embedding = new Embedding <float>(Inputs, cfg.VocabSize, cfg.HiddenSize, initScale: cfg.InitScale); // add dropout EmbeddedOutput = Embedding.Output; if (isTraining && cfg.KeepProb < 1.0) { var dropout = new Dropout <float>(EmbeddedOutput, dropoutProb: 1.0 - cfg.KeepProb); EmbeddedOutput = dropout.Output; } // rnn layer, dropout for intermediate lstm layers and for output if (usingCuDnn) { RnnAccelerated = new Rnn <float>(new LstmRnnType(forgetBiasInit: 0.0), EmbeddedOutput, cfg.NumLayers, cfg.HiddenSize, isTraining: isTraining, dropout: isTraining && cfg.KeepProb < 1.0 ? 1.0 - Config.KeepProb : 0.0); RnnOutput = RnnAccelerated.Y; if (isTraining && cfg.KeepProb < 1.0) { var dropout = new Dropout <float>(RnnOutput, dropoutProb: 1.0 - cfg.KeepProb); RnnOutput = dropout.Output; } } else { RnnDirect = new Lstm <float> [cfg.NumLayers]; for (var i = 0; i < cfg.NumLayers; ++i) { var lstm = new Lstm <float>(i == 0 ? EmbeddedOutput : RnnOutput, cfg.HiddenSize, forgetBiasInit: 0.0); RnnDirect[i] = lstm; RnnOutput = lstm.Y; if (isTraining && cfg.KeepProb < 1.0) { var dropout = new Dropout <float>(RnnOutput, dropoutProb: 1.0 - cfg.KeepProb); RnnOutput = dropout.Output; } } } FC = new FullyConnected <float>(RnnOutput.Reshape(RnnOutput.Shape[0] * RnnOutput.Shape[1], RnnOutput.Shape[2]), cfg.VocabSize); Loss = new SoftmaxCrossEntropySparse <float>(FC.Output, Targets.Reshape(Targets.Shape[0] * Targets.Shape[1])); Optimizer = new GradientDescentOptimizer(ctx, Loss.Loss, cfg.LearningRate, new GlobalNormGradientClipper(cfg.MaxGradNorm)); // warmup to force JIT compilation to get timings without JIT overhead Optimizer.Initalize(); ResetStates(); Optimizer.AssignTensor(Inputs, Fill(Shape.Create(Inputs.Shape.AsArray), 0)); Optimizer.AssignTensor(Targets, Fill(Shape.Create(Targets.Shape.AsArray), 0)); Optimizer.Forward(); if (isTraining) { Optimizer.Backward(); } // now reset states Optimizer.Initalize(); ResetStates(); }
public bool CreateLayer(int nCount, ELayerType type, ActivationSettings activationSettings) { Layer.Utility.Layer layer; switch (type) { case ELayerType.Invalid: throw new ArgumentException("Invalid \"type\" argument."); case ELayerType.AveragePooling: layer = new AveragePooling(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.AverageUnpooling: layer = new AverageUnpooling(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.Convolutional: layer = new Convolutional(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.Deconvolutional: layer = new Deconvolutional(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.Dropout: layer = new Dropout(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.FullyConnected: layer = new FullyConnected(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.GatedRecurrent: layer = new GatedRecurrent(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.LSTM: layer = new LSTM(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.MaxPooling: layer = new MaxPooling(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.MaxUnpooling: layer = new MaxUnpooling(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); case ELayerType.Recurrent: layer = new Recurrent(nCount, Layers.Count, activationSettings); Layers.Add(layer); return(true); default: throw new ArgumentException("Invalid \"type\" argument."); } }
static void Main(string[] args) { /* * // * //1. Test tensor and operations * // * Operations K = new Operations(); * * //Load array to the tensor * NDArray a = new NDArray(3, 6); * a.Load(1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 8, 7, 6, 5, 4, 3, 2, 1); * a.Print("Load array"); * * //Transpose of the matrix * NDArray t = a.Transpose(); * t.Print("Transpose"); * * //Create a tensor with all value 5 * NDArray b = new NDArray(6, 3); * b.Fill(5); * b.Print("Constant 5"); * * //Create a tensor with all value 3 * NDArray c = new NDArray(6, 3); * c.Fill(3); * c.Print("Constant 3"); * * // Subtract two tensor * b = b - c; * * //Perform dot product * NDArray r = K.Dot(a, b); * r.Print("Dot product"); */ // //2. Test layers and activations // /* * //Load array to the tensor * NDArray x = new NDArray(1, 3); * x.Load(1, 2, 3); * x.Print("Load array"); * * //Create two layers, one with 6 neurons and another with 1 * FullyConnected fc1 = new FullyConnected(3, 6, "relu"); * FullyConnected fc2 = new FullyConnected(6, 1, "sigmoid"); * * //Connect input by passing data from one layer to another * fc1.Forward(x); * x = fc1.Output; * x.Print("FC1 Output"); * * fc2.Forward(x); * x = fc2.Output; * x.Print("FC2 Output"); * */ // //3. Test cost functions and metrics // //Load array to the tensor NDArray x = new NDArray(3, 3); x.Load(2, 4, 6, 1, 3, 5, 2, 3, 5); x.Print("Load X values"); NDArray y = new NDArray(3, 1); y.Load(20, 15, 15); y.Print("Load Y values"); //Create two layers one with 6 neurons and another with 1 FullyConnected fc1 = new FullyConnected(3, 6, "relu"); FullyConnected fc2 = new FullyConnected(6, 1, "relu"); //Connect input by passing data from one layer to the other fc1.Forward(x); fc2.Forward(fc1.Output); var preds = fc2.Output; preds.Print("Predictions"); //Calculate mean square error between predicted and expected values BaseCost cost = new BinaryCrossEntropy(); var costValues = cost.Forward(preds, y); costValues.Print("BCE Cost"); //Calculate the mean absolute value for the predicted vs expected values BaseMetric metric = new BinaryAccuracy(); var metricValues = metric.Calculate(preds, y); metricValues.Print("Acc Metric"); //Backpropagation starts here //Calculate gradient cost function var grad = cost.Backward(preds, y); //Then the fc2 layer by passing cost function grad into the layer backward function fc2.Backward(grad); //The grad of the fc2 is stored in the InputGrad property, pass it to fc1 fc1.Backward(fc2.InputGrad); //Print parameters for both layers along with the Grad fc1.PrintParams(); fc2.PrintParams(); }
static void Main(string[] args) { Operations K = new Operations(); //Load array to the tensor NDArray x = new NDArray(3, 3); x.Load(2, 4, 6, 1, 3, 5, 2, 3, 5); x.Print("Load X Values"); NDArray y = new NDArray(3, 1); y.Load(1, 0, 1); y.Print("Load Y Values"); //Create two layers, one with 6 neurons and another with 1 FullyConnected fc1 = new FullyConnected(3, 6, "relu"); FullyConnected fc2 = new FullyConnected(6, 1, "sigmoid"); //Connect input by passing data from one layer to another fc1.Forward(x); fc2.Forward(fc1.Output); var preds = fc2.Output; preds.Print("Predictions"); //Calculate the mean square error cost between the predicted and expected values BaseCost cost = new BinaryCrossEntropy(); var costValues = cost.Forward(preds, y); costValues.Print("BCE Cost"); //Calculate the mean absolute metric value for the predicted vs expected values BaseMetric metric = new BinaryAccuacy(); var metricValues = metric.Calculate(preds, y); metricValues.Print("Acc Metric"); var grad = cost.Backward(preds, y); fc2.Backward(grad); fc1.Backward(fc2.InputGrad); Console.WriteLine("Param value for FC1 before ADAM optimization"); fc1.PrintParams(printGrads: false); //Initialise ADAM optimizer with default learning rate of 0.01 BaseOptimizer optimizer = BaseOptimizer.Get("adam"); //Change the value of learning rate to see the jump is weight changes. //optimizer.LearningRate = 0.1; //Apply optimizer for the first layer for first iteration optimizer.Update(1, fc1); Console.WriteLine("Param value for FC1 after ADAM optimization"); fc1.PrintParams(printGrads: false); Console.WriteLine("Param value for FC2 before ADAM optimization"); fc2.PrintParams(printGrads: false); //Apply optimizer for the first layer for first iteration optimizer.Update(1, fc2); Console.WriteLine("Param value for FC2 after ADAM optimization"); fc2.PrintParams(printGrads: false); Console.ReadLine(); }