/// <summary> /// Clips tensor values to a maximum L2-norm. /// </summary> /// <remarks> /// <para> /// Given a tensor <paramref name="x"/>, and a maximum clip value <paramref name="clip_norm"/>, this operation normalizes /// <paramref name="x"/> so that its L2-norm is less than or equal to <paramref name="clip_norm"/>, along the dimensions /// given in <paramref name="axes"/>. Specifically, in the default case where all dimensions are used for calculation, if /// the L2-norm of <paramref name="x"/> is already less than or equal to <paramref name="clip_norm"/>, then <paramref name="x"/> /// is not modified. If the L2-norm is greater than <paramref name="clip_norm"/>, then this operation returns a tensor of /// the same type and shape as <paramref name="x"/> with its values set to: <c>t* clip_norm / l2norm(t)</c></para> /// </remarks> /// <param name="x">The tensor.</param> /// <param name="clip_norm">The minimum value to clip by. A 0 - D(scalar) tensor, or a tensor with the same shape as <paramref name="x"/>.</param> /// <param name="axes">The minimum value to clip by. A 0 - D(scalar) tensor, or a tensor with the same shape as <paramref name="x"/>.</param> /// <param name="operName">Operation name, optional.</param> /// <returns>A clipped <see cref="TFOutput">tensor</see>.</returns> public TFOutput ClipByNorm(TFOutput x, TFOutput clip_norm, TFOutput?axes = null, string operName = null) { // https://github.com/tensorflow/tensorflow/blob/r1.2/tensorflow/python/ops/clip_ops.py#L73 var scopeName = MakeName("ClipByNorm", operName); using (var newScope = WithScope(scopeName)) { // Calculate L2-norm, clip elements by ratio of clip_norm to L2-norm var l2norm_inv = Rsqrt(ReduceSum(Mul(x, x), axes, keep_dims: true)); var intermediate = Mul(x, clip_norm); var tclip = Identity(Mul(intermediate, Minimum(l2norm_inv, Div(Const(new TFTensor(1.0)), clip_norm), operName: operName))); return(tclip); } }
/// <summary> /// Clips tensor values to a maximum average L2-norm. /// </summary> /// <remarks> /// Given a tensor <paramref name="x"/>, and a maximum clip value <paramref name="clip_norm"/>, this operation /// normalizes <paramref name="x"/> so that its its average L2-norm is less than or equal to <paramref name="clip_norm"/>. /// Specifically, if the average L2-norm is already less than or equal to <paramref name="clip_norm"/>, then <paramref name="x"/> /// is not modified. If the average L2-norm is greater than <paramref name="clip_norm"/>, then this operation returns a tensor of the same /// type and shape as <paramref name="x"/> with its values set to: <c>t* clip_norm / l2norm_avg(t)</c>. In this case, /// the average L2-norm of the output tensor is <paramref name="clip_norm"/>. /// </remarks> /// <param name="x">The input tensor.</param> /// <param name="clip_norm">A maximum clipping value.</param> /// <param name="operName">Name of the oper.</param> public TFOutput ClipByAverageNorm(TFOutput x, TFOutput clip_norm, string operName = null) { // https://github.com/tensorflow/tensorflow/blob/r1.2/tensorflow/python/ops/clip_ops.py#L251 var scopeName = MakeName("ClipByAverageNorm", operName); using (var newScope = WithScope(scopeName)) { // Calculate L2-norm per element, clip elements by ratio of clip_norm to // L2-norm per element TFOutput n_element = Cast(Size(x), TFDataType.Float); TFOutput l2norm_inv = Rsqrt(ReduceSum(Mul(x, x), Range(Rank(x)))); TFOutput tclip = Identity(Mul(Mul(x, clip_norm), Minimum(Mul(l2norm_inv, n_element), Div(Const(new TFTensor(1.0)), clip_norm)), operName: operName)); return(tclip); } }
public void Should_EvaluateBitwiseAnd(int aValue, int bValue, int expected) { using (var graph = new TFGraph()) using (var session = new TFSession(graph)) { TFOutput a = graph.Placeholder(TFDataType.Int32); TFOutput b = graph.Placeholder(TFDataType.Int32); TFOutput y = graph.BitwiseAnd(a, b); TFTensor[] result = session.Run(new[] { a, b }, new TFTensor[] { aValue, bValue }, new[] { y }); Assert.Equal(expected, (int)result[0].GetValue()); } }
public static TFGraph ImConvertGraph(TFTensor image, out TFOutput input, out TFOutput output) { var graph = new TFGraph(); input = graph.Placeholder(TFDataType.String); const int W = 28; const int H = 28; output = graph.Div(graph.ResizeBilinear(graph.ExpandDims(graph.Cast( graph.DecodePng(contents: input, channels: 1), DstT: TFDataType.Float), dim: graph.Const(0)), size: graph.Const(new int[] { W, H }) ), y: graph.Const((float)255) ); return(graph); }
private static void ConstructGraphToNormalizeImage(out TFGraph graph, out TFOutput input, out TFOutput output, TFDataType destinationDataType = TFDataType.Float) { graph = new TFGraph(); input = graph.Placeholder(TFDataType.String); output = graph.Cast(graph.Div( x: graph.Sub( x: graph.ResizeBilinear( images: graph.ExpandDims( input: graph.Cast( graph.DecodeJpeg(contents: input, channels: 3), DstT: TFDataType.Float), dim: graph.Const(0, "make_batch")), size: graph.Const(new int[] { W, H }, "size")), y: graph.Const(Mean, "mean")), y: graph.Const(Scale, "scale")), destinationDataType); }
public void Should_CalculateTanhGrad_Correctly() { using (TFGraph graph = new TFGraph()) using (TFSession session = new TFSession(graph)) { TFOutput x = graph.Const(new TFTensor(0.7)); TFOutput y = graph.Tanh(x); TFOutput dy = graph.Const(new TFTensor(new [] { 1.0 })); TFOutput grad = graph.TanhGrad(y, dy); TFTensor [] result = session.Run(new TFOutput [] { }, new TFTensor [] { }, new [] { grad }); double value = (double)result [0].GetValue(); Assert.Equal(0.634739589982459, value, 15); } }
/// <summary> /// Variable node, with a starting initial value. Convenience that registers the init variable to a global queue. /// </summary> /// <param name="initialValue">Initial value.</param> /// <param name="operName">Operation name, optional.</param> /// <returns>The returning TFOutput returns the handle to the variable.</returns> /// <remarks> /// Variables need to be initialized before the main execution so you will typically want to /// run the session on the variable. /// /// The init sequence for the variable is stored in the graph, you must manually initialize /// those by running the session on the global variables. /// </remarks> public TFOutput Variable(TFOutput initialValue, string operName = null) { var scopeName = MakeName("Variable", operName); using (var newScope = WithScope(scopeName)) { var type = initialValue.OutputType; // This should be VariableV2, but requires Ref support in the C API. var handle = VarHandleOp(type, new TFShape(GetShape(initialValue))); using (var aScope = WithScope("Assign")) { var init = AssignVariableOp(handle, initialValue); AddInitVariable(init); return(handle); } } }
/// <summary> /// Variable node, with a starting initial value. /// </summary> /// <param name="initialValue">Initial value.</param> /// <param name="init">Returns the operation that initializes the value of the variable.</param> /// <param name="value">Returns the value of the variable.</param> /// <param name="operName">Operation name, optional.</param> /// <returns>The returning TFOutput returns the handle to the variable.</returns> /// <remarks> /// Variables need to be initialized before the main execution so you will typically want to /// run the session on the variable /// </remarks> public TFOutput Variable(TFOutput initialValue, out TFOperation init, out TFOutput value, string operName = null) { var scopeName = MakeName("Variable", operName); using (var newScope = WithScope(scopeName)) { var type = initialValue.OutputType; var handle = VarHandleOp(type, new TFShape(GetShape(initialValue))); using (var aScope = WithScope("Assign")) { init = AssignVariableOp(handle, initialValue); using (var rScope = WithScope("Read")) { value = ReadVariableOp(handle, type); return(handle); } } } }
private TFOutput AddActivation(TFGraph graph, TFOutput y, ActivationFunctionType fusedActivationFunction, Layer nonTrivialActivation) { if (fusedActivationFunction != ActivationFunctionType.Linear) { y = graph.AddActivation(y, fusedActivationFunction); } else if (nonTrivialActivation != null) { if (nonTrivialActivation is LeakyRelu leakyRelu) { y = graph.Maximum(y, graph.Mul(y, graph.Const(leakyRelu.Slope))); } } return(y); }
// The inception model takes as input the image described by a Tensor in a very // specific normalized format (a particular image size, shape of the input tensor, // normalized pixel values etc.). // // This function constructs a graph of TensorFlow operations which takes as // input a JPEG-encoded string and returns a tensor suitable as input to the // inception model. // private static TFGraph ConstructGraphToNormalizeImage(out TFOutput input, out TFOutput output, TFDataType destinationDataType = TFDataType.Float) { var graph = new TFGraph(); input = graph.Placeholder(TFDataType.String); output = graph.Cast( graph.ExpandDims( input: graph.Cast(graph.DecodeJpeg(contents: input, channels: 3), DstT: TFDataType.Float), dim: graph.Const(0, "make_batch") ) , destinationDataType ); return(graph); }
public void Should_Stack(double[] x, double[] y, double[] z, int?axis, double[,] expected) { using (var graph = new TFGraph()) using (var session = new TFSession(graph)) { var a = graph.Placeholder(TFDataType.Double, new TFShape(2)); var b = graph.Placeholder(TFDataType.Double, new TFShape(2)); var c = graph.Placeholder(TFDataType.Double, new TFShape(2)); TFOutput r = graph.Stack(new[] { a, b, c }, axis: axis); TFTensor[] result = session.Run(new[] { a, b, c }, new TFTensor[] { x, y, z }, new[] { r }); double[,] actual = (double[, ])result[0].GetValue(); TestUtils.MatrixEqual(expected, actual, precision: 10); } }
public static TFOutput AddActivation(this TFGraph graph, TFOutput input, ActivationFunctionType activation) { switch (activation) { case ActivationFunctionType.Linear: return(input); case ActivationFunctionType.Relu: return(graph.Relu(input)); case ActivationFunctionType.Relu6: return(graph.Relu6(input)); default: throw new NotSupportedException(); } }
public void Should_ClipByValue(double[,] m, double min, double max, double[,] expected) { using (var graph = new TFGraph()) using (var session = new TFSession(graph)) { var matrix = graph.Placeholder(TFDataType.Double); var clip_min = graph.Placeholder(TFDataType.Double); var clip_max = graph.Const(new TFTensor(max)); TFOutput y = graph.ClipByValue(matrix, clip_min, clip_max); TFTensor[] result = session.Run(new[] { matrix, clip_min }, new TFTensor[] { m, min }, new[] { y }); double[,] actual = (double[, ])result[0].GetValue(); Assert.Equal(expected, actual); } }
// Convert the image in filename to a Tensor suitable as input to the Inception model. static TFTensor CreateTensorFromImageFile(byte[] contents) { // DecodeJpeg uses a scalar String-valued tensor as input. var inputTensor = TFTensor.CreateString(contents); TFGraph graph = new TFGraph(); TFOutput input = graph.Placeholder(TFDataType.String); TFOutput output = graph.DecodeJpeg(contents: input, channels: 3); using (var session = new TFSession(graph)) { var tensor = session.Run( inputs: new[] { input }, inputValues: new[] { inputTensor }, outputs: new[] { output }); return(tensor[0]); } }
public ObjectDetector(string graphPath) { using (var graph = new TFGraph()) { var model = File.ReadAllBytes(graphPath); graph.Import(new TFBuffer(model)); this._session = new TFSession(graph); this._detectionInputTensor = graph["image_tensor"][0]; this._detectionOutputTensors = new[] { graph["detection_boxes"][0], graph["detection_scores"][0], graph["detection_classes"][0], // graph["num_detections"][0], }; } }
public void Train(List <TFTensor> inputs, List <TFTensor> expectedOutputs, int numEpochs) { for (int i = 0; i < numEpochs; i++) { int test = Random.Next(0, inputs.Count); TFTensor expectedOutput = expectedOutputs[test]; TFTensor actualOutput = Think(inputs[test]); TFTensor error; using (TFGraph graph = new TFGraph()) { TFOutput expected = graph.Const(expectedOutput); TFOutput actual = graph.Const(actualOutput); TFOutput errorOutput = graph.Sub(expected, actual); TFSession session = new TFSession(graph); error = session.GetRunner().Run(errorOutput); } } }
// Returns range(0, rank(x)) if reduction_indices is null TFOutput ReduceDims (TFOutput input, TFOutput? axis = null) { if (axis.HasValue) return axis.Value; // Fast path: avoid creating Rank and Range ops if ndims is known. var shape = GetTensorShape (input); if (shape.Length >= 0) { // The python code distinguishes between tensor and sparsetensor var array = new int [shape.Length]; for (int i = 0; i < array.Length; i++) array [i] = i; return this.Const (array, TFDataType.Int32); } return Range (Const (0), Const (shape.Length), Const (1)); }
/// <summary> /// Computes the global norm of multiple tensors. /// </summary> /// <remarks> /// <para> /// Given a tuple or list of tensors <paramref name="tensors"/>, this operation returns the global norm of the elements in all tensors /// in <paramref name="tensors"/>. The global norm is computed as: <c>global_norm = sqrt(sum([l2norm(t)**2 for t in t_list]))</c>. Any /// entries in <paramref name="tensors"/> that are of type None are ignored.</para> /// </remarks> /// <param name="tensors">The input tensors.</param> /// <param name="operName">Operation name, optional.</param> /// <returns>A clipped <see cref="TFOutput">tensor</see>.</returns> public TFOutput GlobalNorm(TFOutput [] tensors, string operName = null) { // https://github.com/tensorflow/tensorflow/blob/r1.2/tensorflow/python/ops/clip_ops.py#L122 var scopeName = MakeName("GlobalNorm", operName); using (var newScope = WithScope(scopeName)) { TFOutput [] half_squared_norms = new TFOutput [tensors.Length]; for (int i = 0; i < half_squared_norms.Length; i++) { half_squared_norms [i] = L2Loss(tensors [i]); } TFOutput half_squared_norm = ReduceSum(Stack(half_squared_norms)); TFOutput norm = Sqrt(Mul(half_squared_norm, Const(2.0)), operName: "global_norm"); return(norm); } }
/// <summary> /// Create learning rate time decay operation. /// </summary> protected void CreateDecayOps(float decay, TFOutput initialLearningRate) { if (decay > 0) { var _decay = _graph.Const(decay, "Decay"); var one = _graph.Const(1f); _updateOps.Add(_graph.AssignVariableOp(LearningRate, _graph.Mul(initialLearningRate, _graph.Div(one, _graph.Add(one, _graph.Mul(_decay, _graph.Cast(Iterations.Read, _decay.OutputType) ) ) ) ))); } }
public void TestSession() { var status = new TFStatus(); using (var graph = new TFGraph()) { var feed = Placeholder(graph, status); var two = ScalarConst(2, graph, status); var add = Add(feed, two, graph, status); Assert(status); // Create a session for this graph using (var session = new TFSession(graph, status)) { Assert(status); // Run the graph var inputs = new TFOutput [] { new TFOutput(feed, 0) }; var input_values = new TFTensor [] { 3 }; var outputs = new TFOutput [] { new TFOutput(add, 0) }; var output_values = new TFTensor [] { 3 }; var results = session.Run(runOptions: null, inputs: inputs, inputValues: input_values, outputs: outputs, targetOpers: null, runMetadata: null, status: status); Assert(status); var res = results [0]; Assert(res.TensorType == TFDataType.Int32); Assert(res.NumDims == 0); // Scalar Assert(res.TensorByteSize == (UIntPtr)4); Assert(Marshal.ReadInt32(res.Data) == 3 + 2); } } }
private static TFGraph ConstructGraphToNormalizeRawImage(out TFOutput input, out TFOutput output, int resizeWidth, int resizeHeight, TFDataType destinationDataType = TFDataType.Float) { // Some constants specific to the pre-trained model at: // https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip // // - The model was trained after with images scaled to 224x224 pixels. // - The colors, represented as R, G, B in 1-byte each were converted to // float using (value - Mean)/Scale. const float Mean = 117; const float Scale = 1; //var graph = new TFGraph(); //input = graph.Placeholder(TFDataType.UInt8); //output = graph.Cast(graph.Div( // x: graph.Sub( // x: graph.ResizeBilinear( // images: graph.ExpandDims( // input: graph.Cast(input, DstT: TFDataType.Float), // dim: graph.Const(0, "make_batch")), // size: graph.Const(new int[] { resizeWidth, resizeHeight }, "size")), // y: graph.Const(Mean, "mean")), // y: graph.Const(Scale, "scale")), destinationDataType); //return graph; var graph = new TFGraph(); input = graph.Placeholder(TFDataType.UInt8); output = graph.Cast( graph.ExpandDims( input: graph.Cast(input, DstT: TFDataType.Float), dim: graph.Const(0, "make_batch") ) , destinationDataType ); return(graph); }
public void MNISTTwoHiddenLayerNetworkGPUTest() { // Parameters var learningRate = 0.1f; var epochs = 5; var numGPUs = 4; var mnist = new Mnist(); mnist.ReadDataSets("/tmp"); int batchSize = 400; int numBatches = mnist.TrainImages.Length / batchSize; using (var graph = new TFGraph()) { var X = graph.Placeholder(TFDataType.Float, new TFShape(-1, 784)); var Y = graph.Placeholder(TFDataType.Float, new TFShape(-1, 10)); var Xs = graph.Split(graph.Const(0), X, numGPUs); var Ys = graph.Split(graph.Const(0), Y, numGPUs); var sgd = new SGD(graph, learningRate, 0.9f); TFOutput[] costs = new TFOutput[numGPUs]; TFOutput[] accuracys = new TFOutput[numGPUs]; var variablesAndGradients = new Dictionary <Variable, List <TFOutput> >(); for (int i = 0; i < numGPUs; i++) { using (var device = graph.WithDevice("/GPU:" + i)) { (costs[i], _, accuracys[i]) = CreateNetwork(graph, Xs[i], Ys[i], variablesAndGradients); foreach (var gv in sgd.ComputeGradient(costs[i], colocateGradientsWithOps: true)) { if (!variablesAndGradients.ContainsKey(gv.variable)) { variablesAndGradients[gv.variable] = new List <TFOutput>(); } variablesAndGradients[gv.variable].Add(gv.gradient); } } } var cost = graph.ReduceMean(graph.Stack(costs)); var accuracy = graph.ReduceMean(graph.Stack(accuracys)); var gradientsAndVariables = new (TFOutput gradient, Variable variable)[variablesAndGradients.Count];
static void Main(string[] args) { using (var session = new TFSession()) { TFGraph graph = session.Graph; TFOutput a = graph.Const(2); TFOutput b = graph.Const(3); Console.WriteLine("a=2, b=3"); // 상수 더하기 TFTensor addProcess = session.GetRunner().Run(graph.Add(a, b)); string Result = addProcess.GetValue().ToString(); Console.WriteLine("a+b={0}", Result); // 상수 곱하기 TFTensor multiProcess = session.GetRunner().Run(graph.Mul(a, b)); Result = multiProcess.GetValue().ToString(); Console.ReadLine(); } }
/// <summary> /// Computes dropout. /// </summary> /// <param name="x">A tensor.</param> /// <param name="keep_prob">A scalar Tensor with the same type as x. The probability that each element is kept.</param> /// <param name="noise_shape">A 1-D Tensor of type int32, representing the shape for randomly generated keep/drop flags.</param> /// <param name="seed">Integer seed used for the random distribution, using the TensorFlow SetRandomSeed .</param> /// <param name="operName">Operation name, optional.</param> /// <remarks> /// With probability keep_prob, outputs the input element scaled up by 1 / keep_prob, /// otherwise outputs 0. The scaling is so that the expected sum is unchanged. /// </remarks> public TFOutput Dropout(TFOutput x, double keep_prob, TFShape noise_shape = null, int?seed = null, string operName = null) { if (keep_prob < 0 || keep_prob >= 1) { throw new ArgumentOutOfRangeException("keep_prob must be a scalar tensor or a float in the range (0, 1], got " + keep_prob); } if (keep_prob == 1) { return(x); } var scopeName = MakeName("dropout", operName); using (var newScope = WithScope(scopeName)) { var tkeep_prob = Const(keep_prob); return(Dropout(x, tkeep_prob, noise_shape, seed, operName)); } }
// Returns range(0, rank(x)) if reduction_indices is null TFOutput ReduceDims(TFOutput input, TFOutput?axis = null) { if (axis.HasValue) { return(axis.Value); } // Fast path: avoid creating Rank and Range ops if ndims is known. var shape = GetTensorShape(input); if (shape.IsFullySpecified) { // The python code distinguishes between tensor and sparsetensor return(this.Const(shape.ToIntArray(), TFDataType.Int32)); } // Otherwise, we rely on Range and Rank to do the right thing at run-time. return(Range(Const(0), Rank(input), Const(1))); }
private static TFGraph ConstructGraphToNormalizeImage(int channel, out TFOutput input, out TFOutput output) { var graph = new TFGraph(); input = graph.Placeholder(TFDataType.String); output = //[1 * W * H * RGB / 255] graph.ExpandDims( //[W * H * RGB / 255] graph.Div( //[W * H * RGB] graph.Transpose( //[H * W * RGB] graph.Cast(graph.DecodeBmp(contents: input, channels: channel), DstT: TFDataType.Float) , graph.Const(new int[] { 1, 0, 2 })) , y: graph.Const(255f)) , dim: graph.Const(0)); return(graph); }
public LinearLayer(TFGraph graph, int inSize, int outSize) { var wShape = new TFShape(inSize, outSize); W = graph.VariableV2(wShape, TFDataType.Float); TFOutput initFactor = graph.Sqrt(graph.Div(graph.Const(2F), graph.Const((float)inSize))); TFOutput initialW = graph.Mul(graph.Cast(graph.RandomNormal(wShape, 0, 1), TFDataType.Float), initFactor); InitW = graph.Assign(W, initialW); var bShape = new TFShape(outSize); b = graph.VariableV2(bShape, TFDataType.Float); TFOutput initialB = graph.Zeros(bShape, TFDataType.Float); InitB = graph.Assign(b, initialB); }
public void Apply(TFGraph graph, float learningRate = 0.01F, float beta1 = 0.9F, float beta2 = 0.999F, float eps = 1e-8F) { TFOutput tfLearningRate = graph.Const(learningRate); TFOutput tfBeta1 = graph.Const(beta1); TFOutput tfBeta2 = graph.Const(beta2); TFOutput tfEps = graph.Const(eps); TFOutput tfBeta1Power = graph.Const(beta1); TFOutput tfBeta2Power = graph.Const(beta2); for (int i = 0; i < _variables.Count; i++) { Operations.Add(graph.ApplyAdam(_variables[i], _m[i], _v[i], tfBeta1Power, tfBeta2Power, tfLearningRate, tfBeta1, tfBeta2, tfEps, _gradients[i], null, true).Operation); } }
public IModelResult Eval(IFeatures feature) { var inputValuesTensors = feature.GetFeaturesTensors(); TFOutput output = new TFOutput(); TFOutput input = new TFOutput(); lock (_graph) { using (var session = new TFSession(Graph)) { var result = session.Run( inputs: new[] { input }, inputValues: inputValuesTensors.ToArray(), outputs: new[] { output }); return(ResultConvertor.Convert(result)); } } }
public void tf_random_normal_test() { using (var graph = new TFGraph()) using (var session = new TFSession(graph)) { TFOutput y = graph.RandomNormal(new TFShape(1000, 100), mean: 42, stddev: 0.4, seed: 1337); TFTensor[] result = session.Run(new TFOutput[] { }, new TFTensor[] { }, new[] { y }); object output = result[0].GetValue(); double[,] actual = (double[, ])output; Assert.AreEqual(1000, actual.Rows()); Assert.AreEqual(100, actual.Columns()); double actualMean = actual.Mean(); double expectedMean = 42; Assert.AreEqual(expectedMean, actualMean, 1e-2); } }