public void fit(double[][] input, double[][] target)
{
if (input.Length != target.Length)
{
throw new Exception("Input and target lengths differ");
}
Console.WriteLine("Partially fitting TF model on [{0}] inputs.", input.Length);
int[] order = Enumerable.Range(0, input.Length).ToArray();
for (int i = 0; i < trainingIterations; i++)
{
shuffleArray(order, new Random(random.Next()));
for (int j = 0; j < (target.Length / batchSize) + 1; j++)
{
fillbatch(j, order, input, target);
TFTensor tfInputBatch = TFTensor.FromBuffer(new TFShape(new long[] { batchSize, inputDimension }), trainInputBatch, 0, batchSize * inputDimension);
TFTensor tfOutputBatch = TFTensor.FromBuffer(new TFShape(new long[] { batchSize, outputDimension }), trainOutputBatch, 0, batchSize * outputDimension);
session
.GetRunner()
.AddInput("input_node", tfInputBatch)
.AddInput("output_node", tfOutputBatch)
.AddTarget("train_node")
.Run();
}
}
}
/*
* transform input image to tensor with shape: [1, widht, height, 3]
*/
public static TFTensor TransformInput(Color32[] pic, int width, int height)
{
float[] floatValues = new float[width * height * 3];
// order of image file which read by python cv.imread is BGR
// and left -> right and top -> bottom
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
var color = pic[y * width + x];
int i = (height - y - 1) * width + x; // for change bottom and top
// normalize pixel value ( pixel_value = ((pixel_value - mean_pixel_value) / 255))
floatValues[i * 3 + 0] = (float)((color.b - MEAN_PIXEL_BGR[0]) / IMAGE_STD);
floatValues[i * 3 + 1] = (float)((color.g - MEAN_PIXEL_BGR[1]) / IMAGE_STD);
floatValues[i * 3 + 2] = (float)((color.r - MEAN_PIXEL_BGR[2]) / IMAGE_STD);
}
}
TFShape shape = new TFShape(1, width, height, 3);
return(TFTensor.FromBuffer(shape, floatValues, 0, floatValues.Length));
}
public static TFTensor TensorFromBitmap(Bitmap image, ImageDims dims)
{
int INPUT_SIZE = dims.InputSize;
int IMAGE_MEAN = dims.ImageMean;
float IMAGE_STD = dims.ImageStd;
var bitmap = new Bitmap(image, INPUT_SIZE, INPUT_SIZE);
Color[] colors = new Color[bitmap.Size.Width * bitmap.Size.Height];
int z = 0;
for (int y = bitmap.Size.Height - 1; y >= 0; y--)
{
for (int x = 0; x < bitmap.Size.Width; x++)
{
colors[z] = bitmap.GetPixel(x, y);
z++;
}
}
float[] floatValues = new float[(INPUT_SIZE * INPUT_SIZE) * 3];
for (int i = 0; i < colors.Length; i++)
{
var color = colors[i];
floatValues[i * 3] = (color.R - IMAGE_MEAN) / IMAGE_STD;
floatValues[i * 3 + 1] = (color.G - IMAGE_MEAN) / IMAGE_STD;
floatValues[i * 3 + 2] = (color.B - IMAGE_MEAN) / IMAGE_STD;
}
TFShape shape = new TFShape(1, INPUT_SIZE, INPUT_SIZE, 3);
return(TFTensor.FromBuffer(shape, floatValues, 0, floatValues.Length));
}
/// <summary>
/// Make a prediction for a input image
/// </summary>
/// <param name="image">the input image</param>
/// <returns></returns>
public int Predict(float[,] im)
{
float[] image = TransformArray(im);
//create a runner with the session
var runner = _session.GetRunner();
// create tensor using image
var tensor = TFTensor.FromBuffer(new TFShape(1, 30, 30, 1), image, 0, image.Length);
// set inputput layer
runner.AddInput(_session.Graph[INPUT_NAME][0], tensor);
// set output layer
runner.Fetch(_session.Graph[OUTPUT_NAME][0]);
// run the model
var output = runner.Run();
// Get output tensor
TFTensor result = output[0];
// get result from tensor
var resultArray = result.GetValue() as float[, ];
//return
return(ExtractPrediction(resultArray));
}
private TFTensor GetWrittenDigit(int size)
{
RenderTargetBitmap b = new RenderTargetBitmap(
(int)inkCanvas.ActualWidth, (int)inkCanvas.ActualHeight,
96d, 96d, PixelFormats.Default
);
b.Render(inkCanvas);
var bitmap = new WriteableBitmap(b)
.Resize(size, size, WriteableBitmapExtensions.Interpolation.Bilinear);
float[] data = new float[size * size];
for (int x = 0; x < bitmap.PixelWidth; x++)
{
for (int y = 0; y < bitmap.PixelHeight; y++)
{
var color = bitmap.GetPixel(x, y);
data[y * bitmap.PixelWidth + x] = 255 - ((color.R + color.G + color.B) / 3);
}
}
// sanity check
Console.Write(Stringify(data));
// normalize
for (int i = 0; i < data.Length; i++)
{
data[i] /= 255;
}
return(TFTensor.FromBuffer(new TFShape(1, data.Length), data, 0, data.Length));
}
public static TFTensor GetImageTensor(Bitmap image, int imgWidth, int imgHeight)
{
int channels = 3;
// System.out.println("width: " + imgWidth + ", height: " + imgHeight);
// Generate image file to array
int index = 0;
float[] fb = new float[imgWidth * imgHeight * channels];
// Convert image file to multi-dimension array
for (int row = 0; row < imgHeight; row++)
{
for (int column = 0; column < imgWidth; column++)
{
Color pixel = image.GetPixel(column, row);
float red = pixel.R;
float green = pixel.G;
float blue = pixel.B;
fb[index++] = red;
fb[index++] = green;
fb[index++] = blue;
}
}
return(TFTensor.FromBuffer(new TFShape(new long[] { 1, imgHeight, imgWidth, channels }), fb, 0, fb.Length));
}
public static TFTensor GetFloatToTensor(float[] floatValues, int y, int x, int z)
{
TFShape shape = new TFShape(1, y, x, z);
var tensor = TFTensor.FromBuffer(shape, floatValues, 0, floatValues.Length);
return(tensor);
}
public double[][] predict(double[][] input)
{
double[] buffer = new double[input.Length * inputDimension];
for (int i = 0; i < input.Length; i++)
{
for (int j = 0; j < inputDimension; j++)
{
buffer[i * inputDimension + j] = input[i][j];
}
}
var inputShape = new TFShape(new long[] { input.Length, inputDimension });
TFTensor tfInput = TFTensor.FromBuffer(inputShape, buffer, 0, input.Length * inputDimension);
TFTensor[] output = session
.GetRunner()
.AddInput("input_node", tfInput)
.Fetch("prediction_node_2")
.Run();
TFTensor theFirstAndOnlyFetchedNode = output[0];
bool jagged = true;
object native_array_representation = theFirstAndOnlyFetchedNode.GetValue(jagged); // when jagged = true, returning expected dimensions of return
double[][] typed = (double[][])native_array_representation; // typing to expected array with expected dimensions
return(typed);
}
public static void SVD(float[,] covMat, out float[] s, out float[,] v)
{
TFShape shape = new TFShape(covMat.GetLength(0), covMat.GetLength(1));
var reshaped = covMat.Reshape();
var inputTensor = TFTensor.FromBuffer(shape, reshaped, 0, reshaped.Length);
TFGraph svdGraph = new TFGraph();
TFOutput input = svdGraph.Placeholder(TFDataType.Float, shape);
var svdResult = (ValueTuple <TFOutput, TFOutput, TFOutput>)svdGraph.Svd(input, true);
var sess = new TFSession(svdGraph);
var runner = sess.GetRunner();
runner.AddInput(input, inputTensor);
runner.Fetch(svdResult.Item1);
runner.Fetch(svdResult.Item2);
TFTensor[] results = runner.Run();
s = (float[])results[0].GetValue();
v = (float[, ])results[1].GetValue();
TFStatus temp = new TFStatus();
sess.CloseSession(temp);
sess.DeleteSession(temp);
}
public float[] Predict(float[] input)
{
var tensor = TFTensor.FromBuffer(SingleShape, input, 0, input.Length);
var ouptut = (float[][])Predict(tensor).GetValue(true);
return(ouptut[0]); // Batch size is 1
}
public static TFTensor ToTFTensor <T>(this Tensor <T> tensor)
{
var dims = tensor.Dimensions;
var shapeArr = new long[dims.Length];
for (int i = 0; i < dims.Length; i++)
{
shapeArr[i] = dims[i];
}
var shape = new TFShape(shapeArr);
var count = tensor.Dimensions.GetSize();
var buffer = (object)tensor.ToDenseTensor().Buffer.ToArray();
if (typeof(T) == typeof(int))
{
return(TFTensor.FromBuffer(shape, (int[])buffer, 0, count));
}
else if (typeof(T) == typeof(float))
{
return(TFTensor.FromBuffer(shape, (float[])buffer, 0, count));
}
else
{
throw new NotSupportedException();
}
}
public static TFTensor TransformInput(Color32[] pic, int width, int height)
{
float[] floatValues = new float[width * height * 3];
for (int i = 0; i < pic.Length; ++i)
{
var color = pic[i];
floatValues[i * 3 + 0] = (color.r - 127.5f) / 127.0f;
floatValues[i * 3 + 1] = (color.g - 127.5f) / 127.0f;
floatValues[i * 3 + 2] = (color.b - 127.5f) / 127.0f;
}
// save image
//Texture2D target = new Texture2D(224, 224);
//target.SetPixels32();
//target.Apply();
//var img = target.EncodeToJPG();
//System.IO.File.WriteAllBytes(Application.dataPath + "/captureImage.jpg", img);
//Debug.Log("saved at: " + Application.dataPath);
TFShape shape = new TFShape(1, width, height, 3);
return(TFTensor.FromBuffer(shape, floatValues, 0, floatValues.Length));
}
static void Main(string[] args)
{
FileStream fs = new FileStream("E:\\SVN\\DeepLearning\\MNIST\\train-images.idx3-ubyte", FileMode.Open);
fs.Position = 16;
byte[] content = new byte[fs.Length - 16];
fs.Read(content, 0, content.Length);
fs.Close();
var loaded = TFTensor.FromBuffer(new TFShape(60000, 28, 28, 1), content, 0, content.Length);
using (var session = new TFSession())
{
var graph = session.Graph;
TFOutput input;
TFOutput output;
var(handle, flow) = graph.TensorArrayV3(graph.Const(1), TFDataType.Float, new TFShape(28, 28, 1), true);
input = graph.Placeholder(TFDataType.UInt8, new TFShape(60000, 28, 28, 1));
output = graph.TensorArraySplitV3(handle, graph.Div(graph.Cast(input, TFDataType.Float), graph.Const(255f)), graph.Const(new long[] { 60000L }), flow); //output = handle;
//output = graph.Cast(input, TFDataType.Float);
var result = session.Run(new TFOutput[] { input }, new TFTensor[] { loaded }, new TFOutput[] { output });
output = graph.TensorArrayReadV3(handle, graph.Const(0), flow, TFDataType.Float);
result = session.Run(new TFOutput[] { }, new TFTensor[] { }, new TFOutput[] { output });
}
}
// Convert the image in filename to a Tensor suitable as input to the Inception model.
public static TFTensor CreateTensorFromImageFile(float[] contents, ShapeEnum resolution, TFDataType destinationDataType = TFDataType.Int32)
{
TFShape shape;
if (resolution == ShapeEnum.TEN_TEN)
{
shape = new TFShape(1, 10, 10, 1);
}
else if (resolution == ShapeEnum.THIRTEEN_TEN)
{
shape = new TFShape(1, 13, 10, 1);
}
else if (resolution == ShapeEnum.EIGHT_EIGHT)
{
shape = new TFShape(1, 8, 8, 1);
}
else
{
shape = new TFShape(1, 10, 10, 1);
}
// DecodeJpeg uses a scalar String-valued tensor as input.
var tensor = TFTensor.FromBuffer(shape, contents, 0, contents.Length);
return(tensor);
}
public static TFTensor CreateTensorFromImageFile(string file)
{
var b = new Bitmap(file, true);
var bitmap = new Bitmap(b, INPUT_SIZE, INPUT_SIZE);
Color[] colors = new Color[bitmap.Size.Width * bitmap.Size.Height];
int z = 0;
for (int y = bitmap.Size.Height - 1; y >= 0; y--)
{
for (int x = 0; x < bitmap.Size.Width; x++)
{
colors[z] = bitmap.GetPixel(x, y);
z++;
}
}
float[] floatValues = new float[(INPUT_SIZE * INPUT_SIZE) * 3];
for (int i = 0; i < colors.Length; i++)
{
var color = colors[i];
floatValues[i * 3] = (color.R - IMAGE_MEAN) / IMAGE_STD;
floatValues[i * 3 + 1] = (color.G - IMAGE_MEAN) / IMAGE_STD;
floatValues[i * 3 + 2] = (color.B - IMAGE_MEAN) / IMAGE_STD;
}
TFShape shape = new TFShape(1, INPUT_SIZE, INPUT_SIZE, 3);
return(TFTensor.FromBuffer(shape, floatValues, 0, floatValues.Length));
}
public void CreateTenso()
{
// Some input matrices
var inp = new float[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 35, 55, 82, 115, 135, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 77, 91, 106, 122, 136, 138, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 103, 95, 107, 121, 131, 131, 111, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 112, 101, 91, 102, 121, 126, 126, 108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 107, 96, 88, 90, 108, 125, 130, 138, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 98, 88, 91, 96, 101, 120, 138, 146, 185, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 96, 93, 91, 92, 98, 111, 135, 153, 172, 221, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 91, 93, 100, 107, 125, 153, 185, 210, 226, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 126, 131, 140, 163, 188, 206, 216, 223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 180, 188, 196, 202, 208, 216, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 203, 193, 200, 210, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 232, 222, 216, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 247, 242, 236, 246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 155, 0, 183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
Debug.WriteLine("TEST");
for (int i = 0; i < inp.Length; i++)
{
if (inp[i] == 0)
{
inp[i] = 255;
}
// inp[i] = inp[i] / 255;
//Debug.WriteLine(inp[i]);
}
// Debug.WriteLine("len : " + inp.Length);
var tensor = TFTensor.FromBuffer(new TFShape(1, 30, 30, 1), inp, 0, inp.Length);
// Debug.WriteLine("input: " + tensor);
using (var graph = new TFGraph())
{
// Load the model
// graph.Import(File.ReadAllBytes(@"C:\Users\Public\TestFolder\my_model.pb"));
string model_file = Path.Combine(Environment.CurrentDirectory, @"model\gesture_model.pb");
Debug.WriteLine("file path " + model_file);
graph.Import(File.ReadAllBytes(model_file));
using (var session = new TFSession(graph))
{
var runner = session.GetRunner();
//Debug.WriteLine(graph);
runner.AddInput(graph[INPUT_NAME][0], tensor);
runner.Fetch(graph[OUTPUT_NAME][0]);
Debug.WriteLine(System.DateTime.Now);
var output = runner.Run();
TFTensor result = output[0];
var re = result.GetValue() as float[, ];
foreach (float f in re)
{
Debug.WriteLine(f);
}
Debug.WriteLine("o: " + re[0, 1]);
Debug.WriteLine(result);
Debug.WriteLine(System.DateTime.Now);
}
}
}
public void LoadTFModel(Texture2D tex)
{
var shape = new TFShape(1, inputWidth, inputHeight, 3);
var input = graph[inputName][0];
TFTensor inputTensor = null;
int angle = 90;
Flip flip = Flip.NONE;
if (input.OutputType == TFDataType.Float)
{
float[] imgData = Utils.DecodeTexture(tex, inputWidth, inputHeight,
inputMean, inputStd, angle, flip);
inputTensor = TFTensor.FromBuffer(shape, imgData, 0, imgData.Length);
}
else if (input.OutputType == TFDataType.UInt8)
{
byte[] imgData = Utils.DecodeTexture(tex, inputWidth, inputHeight, angle, flip);
inputTensor = TFTensor.FromBuffer(shape, imgData, 0, imgData.Length);
}
else
{
throw new Exception($"Input date type {input.OutputType} is not supported.");
}
var runner = session.GetRunner();
runner.AddInput(input, inputTensor).Fetch(graph[outputName][0]);
var output = runner.Run()[0];
var outputs = output.GetValue() as float[, ];
inputTensor.Dispose();
output.Dispose();
var list = new List <KeyValuePair <string, float> >();
for (int i = 0; i < labels.Length; i++)
{
var confidence = outputs[0, i];
if (confidence < 0.05f)
{
continue;
}
list.Add(new KeyValuePair <string, float>(labels[i], confidence));
}
var results = list.OrderByDescending(i => i.Value).Take(3).ToList();
foreach (KeyValuePair <string, float> value in results)
{
Debug.Log("my model: " + value.Key);
}
}
private TFTensor GenerateTensor(byte[] image)
{
#if UNITY_ANDROID
TFShape tshape = new TFShape(1, 128, 128, 3);
return(TFTensor.FromBuffer(tshape, image, 0, image.Length));
#endif
#if UNITY_EDITOR_WIN
// TODO: This dosen't work
return(ImageUtil.CreateTensorFromImageFile(image));
#endif
}
private static (TFTensor, TFTensor) CreateTestTensors()
{
var x = new int[] {
1061, 19693, 974, 980, 981, 811, 11474, 589, 39, 1058, 39, 57,
1057, 4934, 46, 1, 1, 1, 1, 1, 1, 1, 1
};
var l = new int[] { 15 };
var tensor = TFTensor.FromBuffer(new TFShape(1, x.Length), x, 0, x.Length);
var lengths = TFTensor.FromBuffer(new TFShape(1), l, 0, l.Length);
return(tensor, lengths);
}
public IList Classify(Texture2D texture, int numResults = 5, float threshold = 0.1f,
int angle = 0, Flip flip = Flip.NONE)
{
var shape = new TFShape(1, inputWidth, inputHeight, 3);
var input = graph[inputName][0];
TFTensor inputTensor = null;
if (input.OutputType == TFDataType.Float)
{
float[] imgData = Utils.DecodeTexture(texture, inputWidth, inputHeight,
inputMean, inputStd, angle, flip);
inputTensor = TFTensor.FromBuffer(shape, imgData, 0, imgData.Length);
}
else if (input.OutputType == TFDataType.UInt8)
{
byte[] imgData = Utils.DecodeTexture(texture, inputWidth, inputHeight, angle, flip);
inputTensor = TFTensor.FromBuffer(shape, imgData, 0, imgData.Length);
}
else
{
throw new Exception($"Input date type {input.OutputType} is not supported.");
}
var runner = session.GetRunner();
runner.AddInput(input, inputTensor).Fetch(graph[outputName][0]);
var output = runner.Run()[0];
var outputs = output.GetValue() as float[, ];
inputTensor.Dispose();
output.Dispose();
var list = new List <KeyValuePair <string, float> >();
for (int i = 0; i < labels.Length; i++)
{
var confidence = outputs[0, i];
if (confidence < threshold)
{
continue;
}
list.Add(new KeyValuePair <string, float>(labels[i], confidence));
}
var results = list.OrderByDescending(i => i.Value).Take(numResults).ToList();
//Utils.Log(results);
return(results);
}
void ThreadedWork()
{
while (true)
{
if (pixelsUpdated)
{
TFShape shape = new TFShape(1, INPUT_SIZE, INPUT_SIZE, 3);
//create a tensor from the image feed from webcam
var tensor = TFTensor.FromBuffer(shape, pixels, 0, pixels.Length);
//run a session with the graph.
var runner = session.GetRunner();
runner.AddInput(graph["image_tensor"][0], tensor).Fetch(
graph["detection_boxes"][0],
graph["detection_scores"][0],
graph["num_detections"][0],
graph["detection_classes"][0]);
output = runner.Run();
var boxes = (float[, , ])output[0].GetValue(jagged: false);
var scores = (float[, ])output[1].GetValue(jagged: false);
var num = (float[])output[2].GetValue(jagged: false);
//int num = 2;
var classes = (float[, ])output[3].GetValue(jagged: false);
items.Clear();
//loop through all detected objects
for (int i = 0; i < num.Length; i++)
{
//for (int i = 0; i < num; i++){
for (int j = 0; j < scores.GetLength(i); j++)
{
float score = scores[i, j];
if (score > MIN_SCORE)
{
CatalogItem catalogItem = _catalog.FirstOrDefault(item => item.Id == Convert.ToInt32(classes[i, j]));
catalogItem.Score = score;
float ymin = boxes[i, j, 0] * Screen.height;
float xmin = boxes[i, j, 1] * Screen.width;
float ymax = boxes[i, j, 2] * Screen.height;
float xmax = boxes[i, j, 3] * Screen.width;
catalogItem.Box = Rect.MinMaxRect(xmin, Screen.height - ymax, xmax, Screen.height - ymin);
items.Add(catalogItem);
Debug.Log(catalogItem.DisplayName);
}
}
}
pixelsUpdated = false;
}
}
}
public List <DetectionResult> RunDetection(int imgWidth, int imgHeight, byte[] img)
{
Debug.Assert(img.Length == imgWidth * imgHeight * 3);
TFTensor tensor = TFTensor.FromBuffer(new TFShape(1, imgWidth, imgHeight, 3), img, 0, img.Length);
TFTensor[] output;
lock (_sessionLocker)
{
output = _session.Run(new[] { _graph["image_tensor"][0] }, new[] { tensor }, new[]
{
_graph["detection_boxes"][0],
_graph["detection_scores"][0],
_graph["detection_classes"][0],
_graph["num_detections"][0]
});
}
var boxes = (float[, , ])output[0].GetValue();
var scores = (float[, ])output[1].GetValue();
var classes = (float[, ])output[2].GetValue();
var xsize = boxes.GetLength(0);
var ysize = Math.Min(boxes.GetLength(1), 5); //HACK: too many results
var results = new List <DetectionResult>();
for (var i = 0; i < xsize; i++)
{
for (var j = 0; j < ysize; j++)
{
var top = (int)(boxes[i, j, 0] * imgHeight);
var left = (int)(boxes[i, j, 1] * imgWidth);
var bottom = (int)(boxes[i, j, 2] * imgHeight);
var right = (int)(boxes[i, j, 3] * imgWidth);
float score = scores[i, j];
var @class = Convert.ToInt32(classes[i, j]);
//if (score < 0.03) break; //HACK: too many results
results.Add(new DetectionResult
{
Box = new BBox(top, left, bottom, right),
Score = score,
Class = @class
});
}
}
return(results);
}
public double[] predict(double[] input)
{
TFTensor tfInput = TFTensor.FromBuffer(new TFShape(new long[] { 1, input.Length }), input, 0, input.Length);
TFTensor[] output = session
.GetRunner()
.AddInput("input_node", tfInput)
.Fetch("prediction_node_2")
.Run();
TFTensor theFirstAndOnlyFetchedNode = output[0];
bool jagged = true;
object native_array_representation = theFirstAndOnlyFetchedNode.GetValue(jagged); // when jagged = true, returning expected dimensions of return
double[][] typed = (double[][])native_array_representation; // typing to expected array with expected dimensions
return(typed[0]); // only one input. so only first output is relevant
}
public static TFTensor TransformInput(Color32[] pic, int width, int height)
{
float[] floatValues = new float[width * height * 3];
for (int i = 0; i < pic.Length; ++i)
{
var color = pic[i];
floatValues[i * 3 + 0] = (color.r - IMAGE_MEAN) / IMAGE_STD;
floatValues[i * 3 + 1] = (color.g - IMAGE_MEAN) / IMAGE_STD;
floatValues[i * 3 + 2] = (color.b - IMAGE_MEAN) / IMAGE_STD;
}
TFShape shape = new TFShape(1, width, height, 3);
return(TFTensor.FromBuffer(shape, floatValues, 0, floatValues.Length));
}
public TFTensor TransformInput(Color32[] pixels)
{
byte[] floatValues = new byte[_imageSize * _imageSize * 3];
for (int i = 0; i < pixels.Length; ++i)
{
var color = pixels[i];
floatValues[i * 3 + 0] = (byte)((color.r - _imageMean) / _imageScale);
floatValues[i * 3 + 1] = (byte)((color.g - _imageMean) / _imageScale);
floatValues[i * 3 + 2] = (byte)((color.b - _imageMean) / _imageScale);
}
var shape = new TFShape(1, _imageSize, _imageSize, 3);
return(TFTensor.FromBuffer(shape, floatValues, 0, floatValues.Length));
}
public int PredictClass(Texture2D image)
{
Debug.Log(image);
float[] imageBytes = new float[image.width * image.height * 3];
int idx = 0;
for (int i = 0; i < image.width; i++)
{
for (int j = 0; j < image.height; j++)
{
Color pixel = image.GetPixel(i, j);
imageBytes[idx++] = pixel.r / 255.0f;
imageBytes[idx++] = pixel.g / 255.0f;
imageBytes[idx++] = pixel.b / 255.0f;
}
}
var runner = session.GetRunner();
runner
.AddInput(graph["conv2d_1_input"][0], TFTensor.FromBuffer(new TFShape(new long[] { 1, 32, 32, 3 }), imageBytes, 0, 3072))
.AddInput(graph["dropout_1/keras_learning_phase"][0], new TFTensor(TFDataType.Bool, new long[0], 1));
runner.Fetch(graph["output_node0"][0]);
float[,] recurrent_tensor = runner.Run()[0].GetValue() as float[, ];
float maxVal = float.MinValue;
int bestIdx = -1;
for (int i = 0; i < recurrent_tensor.GetUpperBound(1); ++i)
{
float val = recurrent_tensor[0, i];
if (val > maxVal)
{
maxVal = val;
bestIdx = i;
}
}
return(bestIdx);
}
public float[] ScoreFIB(string prefix, string postfix, string[] choices)
{
int num_steps = (int)(_params["num_steps"]); // Int32.Parse(_params["num_steps"]);
int num_layers = (int)(_params["num_layers"]); // Int32.Parse(_params["num_layers"]);
int cell_size = (int)(_params["cell_size"]); // Int32.Parse(_params["cell_size"]);
bool lowercase = (bool)(_params["all_lowercase"]); // Boolean.Parse(_params["all_lowercase"]);
int multiplier = 2;
int batch_size = choices.Length;
// generate state with prefix!
// score choices + postfix in a single batch
float[] result = new float[choices.Length];
if (lowercase)
{
prefix = "<s>" + prefix.ToLower();
}
else
{
prefix = "<s>" + prefix;
}
Tuple <float, TFTensor> first_part = ScoreSentenceFast(prefix);
float[] stateTensor = new float[multiplier * cell_size * num_layers * batch_size];
TFShape stateShape = new TFShape(num_layers, multiplier, batch_size, cell_size);
int[] sentenceTensor = new int[num_steps];
TFShape sentenceShape = new TFShape(batch_size, num_steps);
TFTensor state = TFTensor.FromBuffer(stateShape, stateTensor, 0, stateTensor.Length);
int[] smeiTensor = new int[num_steps];
TFShape smeiShape = new TFShape(batch_size, num_steps);
Dictionary <string, TFTensor> inputs = new Dictionary <string, TFTensor>();
string[] outputs = { "output_single_sm", "final_state" };
inputs["state"] = state;
foreach (var choice in choices)
{
string sentence = choices + " " + postfix;
var tokens = Utils.IndexText(_tokenizer.Tokenize(sentence), _vocab).ToArray();
}
return(result);
}
public static TFTensor TransformInput(Color32[] pic, int width, int height)
{
System.Array.Reverse(pic);
float[] floatValues = new float[width * height * 3];
//double sum = 0f;
for (int i = 0; i < pic.Length; ++i)
{
var color = pic[i];
floatValues [i * 3 + 0] = color.r * (2.0f / 255.0f) - 1.0f;
floatValues [i * 3 + 1] = color.g * (2.0f / 255.0f) - 1.0f;
floatValues [i * 3 + 2] = color.b * (2.0f / 255.0f) - 1.0f;
}
TFShape shape = new TFShape(1, width, height, 3);
return(TFTensor.FromBuffer(shape, floatValues, 0, floatValues.Length));
}
public static IrisPrediction Predict(IrisFeatures features)
{
var data = features.ToArray();
var tensor = TFTensor.FromBuffer(new TFShape(1, data.Length), data, 0, data.Length);
using (var graph = new TFGraph())
{
graph.Import(File.ReadAllBytes("keras_frozen.pb"));
var session = new TFSession(graph);
var runner = session.GetRunner();
runner.AddInput(graph["input_layer"][0], tensor);
runner.Fetch(graph["output_layer/Softmax"][0]);
var output = runner.Run();
TFTensor result = output[0];
float[] p = ((float[][])result.GetValue(true))[0];
return(IrisPrediction.FromArray(p));
}
}
public static Tensor MatBgr2Tensor(Mat m, NormalizeMode mode = NormalizeMode.None, int resizeWidth = -1, int resizeHeight = -1, long[] shape = null, float[] imgbuffer = null)
{
if (resizeHeight != -1 && resizeWidth != -1)
{
m.Resize(new Size(resizeWidth, resizeHeight));
}
if (shape == null)
{
shape = new long[] { (int)m.Height, (int)m.Width, m.Channel };
}
float[] buffer = m.GetArray(imgbuffer);
if (mode != NormalizeMode.None)
{
Vector <float> imgBuf = CreateVector.Dense(buffer);
switch (mode)
{
case NormalizeMode.ZeroMean:
imgBuf.Divide(127.5f, imgBuf);
imgBuf.Subtract(1.0f, imgBuf);
break;
case NormalizeMode.ZeroOne:
imgBuf.Divide(255.0f, imgBuf);
break;
case NormalizeMode.CenterZero:
imgBuf.Subtract(imgBuf.Average(), imgBuf);
imgBuf.Divide((float)Statistics.StandardDeviation(imgBuf), imgBuf);
break;
default:
throw new NotImplementedException("unknown one");
}
buffer = imgBuf.Storage.AsArray();
}
TFTensor tensor = TFTensor.FromBuffer(new TFShape(shape), buffer, 0, buffer.Length);
return(new Tensor(tensor));
}