public void TensorFlowTransformInceptionTest()
{
var modelLocation = @"C:\models\TensorFlow\tensorflow_inception_graph.pb";
var mlContext = new MLContext(seed: 1, conc: 1);
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = mlContext.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
var images = new ImageLoaderTransformer(mlContext, imageFolder, ("ImagePath", "ImageReal")).Transform(data);
var cropped = new ImageResizerTransformer(mlContext, "ImageReal", "ImageCropped", 224, 224).Transform(images);
var pixels = new ImagePixelExtractorTransformer(mlContext, "ImageCropped", "input").Transform(cropped);
var tf = new TensorFlowTransformer(mlContext, modelLocation, "input", "softmax2_pre_activation").Transform(pixels);
tf.Schema.TryGetColumnIndex("input", out int input);
tf.Schema.TryGetColumnIndex("softmax2_pre_activation", out int b);
using (var curs = tf.GetRowCursor(col => col == b || col == input))
{
var get = curs.GetGetter <VBuffer <float> >(b);
var getInput = curs.GetGetter <VBuffer <float> >(input);
var buffer = default(VBuffer <float>);
var inputBuffer = default(VBuffer <float>);
while (curs.MoveNext())
{
getInput(ref inputBuffer);
get(ref buffer);
}
}
}
public void TensorFlowTransformCifarInvalidShape()
{
var modelLocation = "cifar_model/frozen_model.pb";
var mlContext = new MLContext(seed: 1, conc: 1);
var imageHeight = 28;
var imageWidth = 28;
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = mlContext.Data.ReadFromTextFile(dataFile,
columns: new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
}
);
var images = new ImageLoaderTransformer(mlContext, imageFolder, ("ImagePath", "ImageReal")).Transform(data);
var cropped = new ImageResizerTransformer(mlContext, "ImageReal", "ImageCropped", imageWidth, imageHeight).Transform(images);
var pixels = new ImagePixelExtractorTransformer(mlContext, "ImageCropped", "Input").Transform(cropped);
var thrown = false;
try
{
IDataView trans = new TensorFlowTransformer(mlContext, modelLocation, "Input", "Output").Transform(pixels);
}
catch
{
thrown = true;
}
Assert.True(thrown);
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only
public void TensorFlowTransformMatrixMultiplicationTest()
{
var modelLocation = "model_matmul/frozen_saved_model.pb";
var mlContext = new MLContext(seed: 1, conc: 1);
// Pipeline
var loader = mlContext.Data.ReadFromEnumerable(
new List <TestData>(new TestData[] {
new TestData()
{
a = new[] { 1.0f, 2.0f,
3.0f, 4.0f },
b = new[] { 1.0f, 2.0f,
3.0f, 4.0f }
},
new TestData()
{
a = new[] { 2.0f, 2.0f,
2.0f, 2.0f },
b = new[] { 3.0f, 3.0f,
3.0f, 3.0f }
}
}));
var trans = new TensorFlowTransformer(mlContext, modelLocation, new[] { "a", "b" }, new[] { "c" }).Transform(loader);
using (var cursor = trans.GetRowCursorForAllColumns())
{
var cgetter = cursor.GetGetter <VBuffer <float> >(2);
Assert.True(cursor.MoveNext());
VBuffer <float> c = default;
cgetter(ref c);
var cValues = c.GetValues();
Assert.Equal(1.0 * 1.0 + 2.0 * 3.0, cValues[0]);
Assert.Equal(1.0 * 2.0 + 2.0 * 4.0, cValues[1]);
Assert.Equal(3.0 * 1.0 + 4.0 * 3.0, cValues[2]);
Assert.Equal(3.0 * 2.0 + 4.0 * 4.0, cValues[3]);
Assert.True(cursor.MoveNext());
c = default;
cgetter(ref c);
cValues = c.GetValues();
Assert.Equal(2.0 * 3.0 + 2.0 * 3.0, cValues[0]);
Assert.Equal(2.0 * 3.0 + 2.0 * 3.0, cValues[1]);
Assert.Equal(2.0 * 3.0 + 2.0 * 3.0, cValues[2]);
Assert.Equal(2.0 * 3.0 + 2.0 * 3.0, cValues[3]);
Assert.False(cursor.MoveNext());
}
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only
public void TensorFlowTransformCifar()
{
var modelLocation = "cifar_model/frozen_model.pb";
var mlContext = new MLContext(seed: 1, conc: 1);
var tensorFlowModel = TensorFlowUtils.LoadTensorFlowModel(mlContext, modelLocation);
var schema = tensorFlowModel.GetInputSchema();
Assert.True(schema.TryGetColumnIndex("Input", out int column));
var type = (VectorType)schema[column].Type;
var imageHeight = type.Dimensions[0];
var imageWidth = type.Dimensions[1];
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = mlContext.Data.ReadFromTextFile(dataFile,
columns: new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
}
);
var pipeEstimator = new ImageLoadingEstimator(mlContext, imageFolder, ("ImagePath", "ImageReal"))
.Append(new ImageResizingEstimator(mlContext, "ImageReal", "ImageCropped", imageWidth, imageHeight))
.Append(new ImagePixelExtractingEstimator(mlContext, "ImageCropped", "Input", interleave: true));
var pixels = pipeEstimator.Fit(data).Transform(data);
IDataView trans = new TensorFlowTransformer(mlContext, tensorFlowModel, "Input", "Output").Transform(pixels);
trans.Schema.TryGetColumnIndex("Output", out int output);
using (var cursor = trans.GetRowCursor(col => col == output))
{
var buffer = default(VBuffer <float>);
var getter = cursor.GetGetter <VBuffer <float> >(output);
var numRows = 0;
while (cursor.MoveNext())
{
getter(ref buffer);
Assert.Equal(10, buffer.Length);
numRows += 1;
}
Assert.Equal(4, numRows);
}
}
public void TensorFlowTransformObjectDetectionTest()
{
var modelLocation = @"C:\models\TensorFlow\ssd_mobilenet_v1_coco_2018_01_28\frozen_inference_graph.pb";
var mlContext = new MLContext(seed: 1, conc: 1);
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = mlContext.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
var images = new ImageLoaderTransformer(mlContext, imageFolder, ("ImagePath", "ImageReal")).Transform(data);
var cropped = new ImageResizerTransformer(mlContext, "ImageReal", "ImageCropped", 32, 32).Transform(images);
var pixels = new ImagePixelExtractorTransformer(mlContext, "ImageCropped", "image_tensor", asFloat: false).Transform(cropped);
var tf = new TensorFlowTransformer(mlContext, modelLocation, new[] { "image_tensor" },
new[] { "detection_boxes", "detection_scores", "num_detections", "detection_classes" }).Transform(pixels);
tf.Schema.TryGetColumnIndex("image_tensor", out int input);
tf.Schema.TryGetColumnIndex("detection_boxes", out int boxes);
tf.Schema.TryGetColumnIndex("detection_scores", out int scores);
tf.Schema.TryGetColumnIndex("num_detections", out int num);
tf.Schema.TryGetColumnIndex("detection_classes", out int classes);
using (var curs = tf.GetRowCursor(tf.Schema["image_tensor"], tf.Schema["detection_boxes"], tf.Schema["detection_scores"], tf.Schema["detection_classes"], tf.Schema["num_detections"]))
{
var getInput = curs.GetGetter <VBuffer <byte> >(input);
var getBoxes = curs.GetGetter <VBuffer <float> >(boxes);
var getScores = curs.GetGetter <VBuffer <float> >(scores);
var getNum = curs.GetGetter <VBuffer <float> >(num);
var getClasses = curs.GetGetter <VBuffer <float> >(classes);
var buffer = default(VBuffer <float>);
var inputBuffer = default(VBuffer <byte>);
while (curs.MoveNext())
{
getInput(ref inputBuffer);
getBoxes(ref buffer);
getScores(ref buffer);
getNum(ref buffer);
getClasses(ref buffer);
}
}
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only
public void TensorFlowTransformInputOutputTypesTest()
{
// This an identity model which returns the same output as input.
var model_location = "model_types_test";
//Data
var data = new List <TypesData>(
new TypesData[] {
new TypesData()
{
f64 = new[] { -1.0, 2.0 },
f32 = new[] { -1.0f, 2.0f },
i64 = new[] { -1L, 2 },
i32 = new[] { -1, 2 },
i16 = new short[] { -1, 2 },
i8 = new sbyte[] { -1, 2 },
u64 = new ulong[] { 1, 2 },
u32 = new uint[] { 1, 2 },
u16 = new ushort[] { 1, 2 },
u8 = new byte[] { 1, 2 },
b = new bool[] { true, true },
},
new TypesData()
{
f64 = new[] { -3.0, 4.0 },
f32 = new[] { -3.0f, 4.0f },
i64 = new[] { -3L, 4 },
i32 = new[] { -3, 4 },
i16 = new short[] { -3, 4 },
i8 = new sbyte[] { -3, 4 },
u64 = new ulong[] { 3, 4 },
u32 = new uint[] { 3, 4 },
u16 = new ushort[] { 3, 4 },
u8 = new byte[] { 3, 4 },
b = new bool[] { false, false },
}
});
var mlContext = new MLContext(seed: 1, conc: 1);
// Pipeline
var loader = ComponentCreation.CreateDataView(mlContext, data);
var inputs = new string[] { "f64", "f32", "i64", "i32", "i16", "i8", "u64", "u32", "u16", "u8", "b" };
var outputs = new string[] { "o_f64", "o_f32", "o_i64", "o_i32", "o_i16", "o_i8", "o_u64", "o_u32", "o_u16", "o_u8", "o_b" };
var trans = new TensorFlowTransformer(mlContext, model_location, inputs, outputs).Transform(loader);;
using (var cursor = trans.GetRowCursor(a => true))
{
var f64getter = cursor.GetGetter <VBuffer <double> >(11);
var f32getter = cursor.GetGetter <VBuffer <float> >(12);
var i64getter = cursor.GetGetter <VBuffer <long> >(13);
var i32getter = cursor.GetGetter <VBuffer <int> >(14);
var i16getter = cursor.GetGetter <VBuffer <short> >(15);
var i8getter = cursor.GetGetter <VBuffer <sbyte> >(16);
var u64getter = cursor.GetGetter <VBuffer <ulong> >(17);
var u32getter = cursor.GetGetter <VBuffer <uint> >(18);
var u16getter = cursor.GetGetter <VBuffer <ushort> >(19);
var u8getter = cursor.GetGetter <VBuffer <byte> >(20);
var boolgetter = cursor.GetGetter <VBuffer <bool> >(21);
VBuffer <double> f64 = default;
VBuffer <float> f32 = default;
VBuffer <long> i64 = default;
VBuffer <int> i32 = default;
VBuffer <short> i16 = default;
VBuffer <sbyte> i8 = default;
VBuffer <ulong> u64 = default;
VBuffer <uint> u32 = default;
VBuffer <ushort> u16 = default;
VBuffer <byte> u8 = default;
VBuffer <bool> b = default;
foreach (var sample in data)
{
Assert.True(cursor.MoveNext());
f64getter(ref f64);
f32getter(ref f32);
i64getter(ref i64);
i32getter(ref i32);
i16getter(ref i16);
i8getter(ref i8);
u64getter(ref u64);
u32getter(ref u32);
u16getter(ref u16);
u8getter(ref u8);
u8getter(ref u8);
boolgetter(ref b);
var f64Values = f64.GetValues();
Assert.Equal(2, f64Values.Length);
Assert.True(f64Values.SequenceEqual(sample.f64));
var f32Values = f32.GetValues();
Assert.Equal(2, f32Values.Length);
Assert.True(f32Values.SequenceEqual(sample.f32));
var i64Values = i64.GetValues();
Assert.Equal(2, i64Values.Length);
Assert.True(i64Values.SequenceEqual(sample.i64));
var i32Values = i32.GetValues();
Assert.Equal(2, i32Values.Length);
Assert.True(i32Values.SequenceEqual(sample.i32));
var i16Values = i16.GetValues();
Assert.Equal(2, i16Values.Length);
Assert.True(i16Values.SequenceEqual(sample.i16));
var i8Values = i8.GetValues();
Assert.Equal(2, i8Values.Length);
Assert.True(i8Values.SequenceEqual(sample.i8));
var u64Values = u64.GetValues();
Assert.Equal(2, u64Values.Length);
Assert.True(u64Values.SequenceEqual(sample.u64));
var u32Values = u32.GetValues();
Assert.Equal(2, u32Values.Length);
Assert.True(u32Values.SequenceEqual(sample.u32));
var u16Values = u16.GetValues();
Assert.Equal(2, u16Values.Length);
Assert.True(u16Values.SequenceEqual(sample.u16));
var u8Values = u8.GetValues();
Assert.Equal(2, u8Values.Length);
Assert.True(u8Values.SequenceEqual(sample.u8));
var bValues = b.GetValues();
Assert.Equal(2, bValues.Length);
Assert.True(bValues.SequenceEqual(sample.b));
}
Assert.False(cursor.MoveNext());
}
}
public Mapper(TensorFlowTransformer parent, DataViewSchema inputSchema) :
base(Contracts.CheckRef(parent, nameof(parent)).Host.Register(nameof(Mapper)), inputSchema, parent)
{
Host.CheckValue(parent, nameof(parent));
_parent = parent;
_inputColIndices = new int[_parent.Inputs.Length];
_isInputVector = new bool[_parent.Inputs.Length];
_fullySpecifiedShapes = new TensorShape[_parent.Inputs.Length];
for (int i = 0; i < _parent.Inputs.Length; i++)
{
if (!inputSchema.TryGetColumnIndex(_parent.Inputs[i], out _inputColIndices[i]))
{
throw Host.ExceptSchemaMismatch(nameof(InputSchema), "source", _parent.Inputs[i]);
}
var type = inputSchema[_inputColIndices[i]].Type;
if (type is VectorDataViewType vecType && vecType.Size == 0)
{
throw Host.Except("Variable length input columns not supported");
}
_isInputVector[i] = type is VectorDataViewType;
if (!_isInputVector[i])
{
throw Host.Except("Non-vector columns are not supported and should be loaded as vector columns of size 1");
}
vecType = (VectorDataViewType)type;
var expectedType = Tf2MlNetType(_parent.TFInputTypes[i]);
if (type.GetItemType() != expectedType)
{
throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", _parent.Inputs[i], expectedType.ToString(), type.ToString());
}
var originalShape = _parent.TFInputShapes[i];
var shape = originalShape.dims;
var colTypeDims = vecType.Dimensions.Select(dim => (int)dim).ToArray();
if (shape == null || (shape.Length == 0))
{
_fullySpecifiedShapes[i] = new TensorShape(colTypeDims);
}
else
{
// If the column is one dimension we make sure that the total size of the TF shape matches.
// Compute the total size of the known dimensions of the shape.
int valCount = 1;
int numOfUnkDim = 0;
foreach (var s in shape)
{
if (s > 0)
{
valCount *= s;
}
else
{
numOfUnkDim++;
}
}
// The column length should be divisible by this, so that the other dimensions can be integral.
int typeValueCount = type.GetValueCount();
if (typeValueCount % valCount != 0)
{
throw Contracts.Except($"Input shape mismatch: Input '{_parent.Inputs[i]}' has shape {originalShape.ToString()}, but input data is of length {typeValueCount}.");
}
// This cover the 2-variable senario e.g. [?, ?, ?, C] where we can assume typeDims provides the information of [W, H, C]
// The shape will become [?, W, H, C]
var originalShapeDims = originalShape.dims;
var originalShapeNdim = originalShape.ndim;
if (numOfUnkDim == 3 && colTypeDims.Length == 3 && originalShapeNdim == numOfUnkDim + 1 && originalShapeDims[1] == -1)
{
originalShapeDims[1] = colTypeDims[0];
originalShapeDims[2] = colTypeDims[1];
valCount *= originalShapeDims[1] * originalShapeDims[2];
numOfUnkDim -= 2;
}
// If the shape is multi-dimensional, we should be able to create the length of the vector by plugging
// in a single value for the unknown shapes. For example, if the shape is [?,?,3], then there should exist a value
// d such that d*d*3 is equal to the length of the input column.
var d = numOfUnkDim > 0 ? Math.Pow(typeValueCount / valCount, 1.0 / numOfUnkDim) : 0;
if (d - (int)d != 0)
{
throw Contracts.Except($"Input shape mismatch: Input '{_parent.Inputs[i]}' has shape {originalShape.ToString()}, but input data is of length {typeValueCount}.");
}
// Fill in the unknown dimensions.
var l = new int[originalShapeNdim];
for (int ishape = 0; ishape < originalShapeNdim; ishape++)
{
l[ishape] = originalShapeDims[ishape] == -1 ? (int)d : originalShapeDims[ishape];
}
_fullySpecifiedShapes[i] = new TensorShape(l);
}
if (_parent._addBatchDimensionInput)
{
var l = new int[_fullySpecifiedShapes[i].ndim + 1];
l[0] = 1;
for (int ishape = 1; ishape < l.Length; ishape++)
{
l[ishape] = _fullySpecifiedShapes[i].dims[ishape - 1];
}
_fullySpecifiedShapes[i] = new TensorShape(l);
}
}
_runners = new ConcurrentBag <Runner>();
}
