public float Compute(ImageDataPoint <float> point)
{
if (point.ImageID != _currentID)
{
_currentID = point.ImageID;
if (point.Image.IsIntegral || DecisionTree <ImageDataPoint <float>, float[]> .IsBuilding)
{
_integralImage = new FloatArrayHandler(point.Image.RawArray, true);
}
else
{
_integralImage = IntegralImage.ComputeFloat <FloatArrayHandler>(point.Image);
}
}
int row = point.Row;
int column = point.Column;
IMultichannelImage <float> image = point.Image;
float sum = 0;
for (int i = 0; i < _count; i++)
{
if (i % 2 == 0)
{
sum += _integralImage.ComputeRectangleSum(row, column, _channels[i], _rectangles[i]);
}
else
{
sum -= _integralImage.ComputeRectangleSum(row, column, _channels[i], _rectangles[i]);
}
}
Debug.Assert(!float.IsNaN(sum), "Rectangle sum is NaN!");
return(sum);
}
/// <summary>
/// Create a new Filter Bank Image. The input image is run through the provided filter banks and their responses are concatenated together to form the channels of the image.
/// </summary>
/// <param name="input">The input image</param>
/// <param name="filterBanks">The filter banks to apply</param>
/// <returns>The image</returns>
public static unsafe FilterBankImage Create(IMultichannelImage <float> input, params FilterBank[] filterBanks)
{
FilterBankImage image = new FilterBankImage();
image.SetDimensions(input.Rows, input.Columns, filterBanks.Sum(o => o.DescriptorLength));
fixed(float *dataSrc = image.RawArray)
{
float *dataPtr = dataSrc;
for (short r = 0; r < input.Rows; r++)
{
for (short c = 0; c < input.Columns; c++)
{
ImageDataPoint <float> point = new ImageDataPoint <float>(input, r, c, 0);
foreach (var fb in filterBanks)
{
float[] values = fb.Compute(point);
for (int i = 0; i < values.Length; i++, dataPtr++)
{
*dataPtr = values[i];
}
}
}
}
}
return(image);
}
public float Compute(ImageDataPoint <float> point)
{
int row = point.Row;
int column = point.Column;
float value1 = GetValue1(row, column, point);
float value2 = GetValue2(row, column, point);
if (value2 == 0)
{
value2 = Decider <ImageDataPoint <float>, float[]> .DIRICHLET_PRIOR;
}
float val = value1 / value2;
if (_absoluteValue)
{
val = Math.Abs(val);
}
#if DEBUG
if (float.IsNegativeInfinity(val) || float.IsNaN(val) || float.IsPositiveInfinity(val))
{
throw new Exception("invalid feature value!");
}
#endif
return(val);
}
/// <summary>
/// Computes the filter bank at the provided point.
/// </summary>
/// <param name="point">Desired location</param>
/// <returns>Filter response</returns>
public virtual float[] Compute(ImageDataPoint <float> point)
{
float[] desc = new float[_filters.Count];
for (int i = 0; i < desc.Length; i++)
{
desc[i] = _filters[i].Compute(point);
}
return(desc);
}
protected float GetValue2(int row, int column, ImageDataPoint <float> point)
{
row += _row2;
column += _column2;
IMultichannelImage <float> image = point.Image;
fixValue(ref row, 0, image.Rows);
fixValue(ref column, 0, image.Columns);
return(image.RawArray[row, column, _channel2]);
}
// Example of applying ONNX transform on in-memory images.
public static void Example()
{
// Download the squeeznet image model from ONNX model zoo, version 1.2
// https://github.com/onnx/models/tree/master/squeezenet or use
// Microsoft.ML.Onnx.TestModels nuget.
// It's a multiclass classifier. It consumes an input "data_0" and produces
// an output "softmaxout_1".
var modelPath = @"squeezenet\00000001\model.onnx";
// Create ML pipeline to score the data using OnnxScoringEstimator
var mlContext = new MLContext();
// Create in-memory data points. Its Image/Scores field is the input/output of the used ONNX model.
var dataPoints = new ImageDataPoint[]
{
new ImageDataPoint(Color.Red),
new ImageDataPoint(Color.Green)
};
// Convert training data to IDataView, the general data type used in ML.NET.
var dataView = mlContext.Data.LoadFromEnumerable(dataPoints);
// Create a ML.NET pipeline which contains two steps. First, ExtractPixle is used to convert the 224x224 image to a 3x224x224 float tensor.
// Then the float tensor is fed into a ONNX model with an input called "data_0" and an output called "softmaxout_1". Note that "data_0" and
// "softmaxout_1" are model input and output names stored in the used ONNX model file. Users may need to inspect their own models to
// get the right input and output column names.
var pipeline = mlContext.Transforms.ExtractPixels("data_0", "Image") // Map column "Image" to column "data_0"
.Append(mlContext.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelPath)); // Map column "data_0" to column "softmaxout_1"
var model = pipeline.Fit(dataView);
var onnx = model.Transform(dataView);
// Convert IDataView back to IEnumerable<ImageDataPoint> so that user can inspect the output, column "softmaxout_1", of the ONNX transform.
// Note that Column "softmaxout_1" would be stored in ImageDataPont.Scores because the added attributed [ColumnName("softmaxout_1")]
// tells that ImageDataPont.Scores is equivalent to column "softmaxout_1".
var transformedDataPoints = mlContext.Data.CreateEnumerable <ImageDataPoint>(onnx, false).ToList();
// The scores are probabilities of all possible classes, so they should all be positive.
foreach (var dataPoint in transformedDataPoints)
{
var firstClassProb = dataPoint.Scores.First();
var lastClassProb = dataPoint.Scores.Last();
Console.WriteLine($"The probability of being the first class is {firstClassProb * 100}%.");
Console.WriteLine($"The probability of being the last class is {lastClassProb * 100}%.");
}
// Expected output:
// The probability of being the first class is 0.002542659%.
// The probability of being the last class is 0.0292684%.
// The probability of being the first class is 0.02258059%.
// The probability of being the last class is 0.394428%.
}
public float Compute(ImageDataPoint <float> point)
{
float[,,] data = point.Image.RawArray;
int rows = data.GetLength(0);
int columns = data.GetLength(1);
int row = point.Row;
int column = point.Column;
float sum = 0;
foreach (Part part in _parts)
{
sum += part.Compute(data, row, column, rows, columns);
}
return(sum);
}
public void OnnxModelInMemoryImage()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
return;
}
// Path of ONNX model. It's a multiclass classifier. It consumes an input "data_0" and produces an output "softmaxout_1".
var modelFile = "squeezenet/00000001/model.onnx";
// Create in-memory data points. Its Image/Scores field is the input/output of the used ONNX model.
var dataPoints = new ImageDataPoint[]
{
new ImageDataPoint(Color.Red),
new ImageDataPoint(Color.Green)
};
// Convert training data to IDataView, the general data type used in ML.NET.
var dataView = ML.Data.LoadFromEnumerable(dataPoints);
// Create a ML.NET pipeline which contains two steps. First, ExtractPixle is used to convert the 224x224 image to a 3x224x224 float tensor.
// Then the float tensor is fed into a ONNX model with an input called "data_0" and an output called "softmaxout_1". Note that "data_0" and
// "softmaxout_1" are model input and output names stored in the used ONNX model file. Users may need to inspect their own models to
// get the right input and output column names.
var pipeline = ML.Transforms.ExtractPixels("data_0", "Image") // Map column "Image" to column "data_0"
.Append(ML.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelFile)); // Map column "data_0" to column "softmaxout_1"
var model = pipeline.Fit(dataView);
var onnx = model.Transform(dataView);
// Convert IDataView back to IEnumerable<ImageDataPoint> so that user can inspect the output, column "softmaxout_1", of the ONNX transform.
// Note that Column "softmaxout_1" would be stored in ImageDataPont.Scores because the added attributed [ColumnName("softmaxout_1")]
// tells that ImageDataPont.Scores is equivalent to column "softmaxout_1".
var transformedDataPoints = ML.Data.CreateEnumerable <ImageDataPoint>(onnx, false).ToList();
// The scores are probabilities of all possible classes, so they should all be positive.
foreach (var dataPoint in transformedDataPoints)
{
foreach (var score in dataPoint.Scores)
{
Assert.True(score > 0);
}
}
}
public float Compute(ImageDataPoint <float> point)
{
if (point.ImageID != _currentID)
{
_currentID = point.ImageID;
if (point.Image.IsIntegral || DecisionTree <ImageDataPoint <float>, float[]> .IsBuilding)
{
_integralImage = point.Image;
}
else
{
_integralImage = IntegralImage.ComputeFloat <FloatArrayHandler>(point.Image);
}
}
int row = point.Row;
int column = point.Column;
float sum = _integralImage.ComputeRectangleSum(row, column, _channel, _rect);
Debug.Assert(!float.IsNaN(sum), "Rectangle sum is NaN");
return(sum);
}
public float Compute(ImageDataPoint <float> point)
{
int row = point.Row + _row;
int column = point.Column + _column;
IMultichannelImage <float> image = point.Image;
fixValue(ref row, 0, image.Rows);
fixValue(ref column, 0, image.Columns);
float val = image.RawArray[row, column, _channel];
if (_useLog)
{
val = Math.Abs(val);
val = Math.Max(val, Decider <ImageDataPoint <float>, float[]> .DIRICHLET_PRIOR);
val = (float)Math.Log(val);
}
if (_useAbsoluteValue)
{
val = Math.Abs(val);
}
return(val);
}
public void TestGrayScaleInMemory()
{
// Create an image list.
var images = new List <ImageDataPoint>()
{
new ImageDataPoint(10, 10, Color.Blue), new ImageDataPoint(10, 10, Color.Red)
};
// Convert the list of data points to an IDataView object, which is consumable by ML.NET API.
var data = ML.Data.LoadFromEnumerable(images);
// Convert image to gray scale.
var pipeline = ML.Transforms.ConvertToGrayscale("GrayImage", "Image");
// Fit the model.
var model = pipeline.Fit(data);
// Test path: image files -> IDataView -> Enumerable of Bitmaps.
var transformedData = model.Transform(data);
// Load images in DataView back to Enumerable.
var transformedDataPoints = ML.Data.CreateEnumerable <ImageDataPoint>(transformedData, false);
foreach (var dataPoint in transformedDataPoints)
{
var image = dataPoint.Image;
var grayImage = dataPoint.GrayImage;
Assert.NotNull(grayImage);
Assert.Equal(image.Width, grayImage.Width);
Assert.Equal(image.Height, grayImage.Height);
for (int x = 0; x < grayImage.Width; ++x)
{
for (int y = 0; y < grayImage.Height; ++y)
{
var pixel = grayImage.GetPixel(x, y);
// greyscale image has same values for R, G and B.
Assert.True(pixel.R == pixel.G && pixel.G == pixel.B);
}
}
}
var engine = ML.Model.CreatePredictionEngine <ImageDataPoint, ImageDataPoint>(model);
var singleImage = new ImageDataPoint(17, 36, Color.Pink);
var transformedSingleImage = engine.Predict(singleImage);
Assert.Equal(singleImage.Image.Height, transformedSingleImage.GrayImage.Height);
Assert.Equal(singleImage.Image.Width, transformedSingleImage.GrayImage.Width);
for (int x = 0; x < transformedSingleImage.GrayImage.Width; ++x)
{
for (int y = 0; y < transformedSingleImage.GrayImage.Height; ++y)
{
var pixel = transformedSingleImage.GrayImage.GetPixel(x, y);
// greyscale image has same values for R, G and B.
Assert.True(pixel.R == pixel.G && pixel.G == pixel.B);
}
}
}
public float Compute(ImageDataPoint <float> point)
{
return(_filter.Compute(point));
}
/// <summary>
/// Computes filter by multiplying the filter values against the patch values. If the patch or filter
/// sizes do not match up, the two will be centered on each other and computation done on overlapping
/// indices.
/// </summary>
/// <param name="point">Point to use when computing filter</param>
/// <returns>Filter response for the desired sample</returns>
public unsafe float Compute(ImageDataPoint <float> point)
{
return(compute(point.Row, point.Column, point.Image));
}
