public void TestTensorFlowWithSchema()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
return;
}
const string modelLocation = "cifar_model/frozen_model.pb";
var mlContext = new MLContext(seed: 1);
var tensorFlowModel = TensorFlowUtils.LoadTensorFlowModel(mlContext, modelLocation);
var schema = tensorFlowModel.GetInputSchema();
Assert.True(schema.TryGetColumnIndex("Input", out int column));
var type = (VectorDataViewType)schema[column].Type;
var imageHeight = type.Dimensions[0];
var imageWidth = type.Dimensions[1];
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = ML.Data.LoadFromTextFile(dataFile, new[] {
new TextLoader.Column("imagePath", DataKind.String, 0),
new TextLoader.Column("name", DataKind.String, 1)
});
// Note that CamelCase column names are there to match the TF graph node names.
var pipe = ML.Transforms.LoadImages("Input", imageFolder, "imagePath")
.Append(ML.Transforms.ResizeImages("Input", imageHeight, imageWidth))
.Append(ML.Transforms.ExtractPixels("Input", interleavePixelColors: true))
.Append(tensorFlowModel.ScoreTensorFlowModel("Output", "Input"));
TestEstimatorCore(pipe, data);
var result = pipe.Fit(data).Transform(data);
result.Schema.TryGetColumnIndex("Output", out int output);
using (var cursor = result.GetRowCursor(result.Schema["Output"]))
{
var buffer = default(VBuffer <float>);
var getter = cursor.GetGetter <VBuffer <float> >(result.Schema["Output"]);
var numRows = 0;
while (cursor.MoveNext())
{
getter(ref buffer);
Assert.Equal(10, buffer.Length);
numRows += 1;
}
Assert.Equal(4, numRows);
}
}
public void TestTensorFlowStaticWithSchema()
{
const string modelLocation = "cifar_model/frozen_model.pb";
var mlContext = new MLContext(seed: 1, conc: 1);
var tensorFlowModel = TensorFlowUtils.LoadTensorFlowModel(mlContext, modelLocation);
var schema = tensorFlowModel.GetInputSchema();
Assert.True(schema.TryGetColumnIndex("Input", out int column));
var type = (VectorType)schema[column].Type;
var imageHeight = type.Dimensions[0];
var imageWidth = type.Dimensions[1];
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = TextLoaderStatic.CreateLoader(mlContext, ctx => (
imagePath: ctx.LoadText(0),
name: ctx.LoadText(1)))
.Load(dataFile);
// Note that CamelCase column names are there to match the TF graph node names.
var pipe = data.MakeNewEstimator()
.Append(row => (
row.name,
Input: row.imagePath.LoadAsImage(imageFolder).Resize(imageHeight, imageWidth).ExtractPixels(interleave: true)))
.Append(row => (row.name, Output: row.Input.ApplyTensorFlowGraph(tensorFlowModel)));
TestEstimatorCore(pipe.AsDynamic, data.AsDynamic);
var result = pipe.Fit(data).Transform(data).AsDynamic;
result.Schema.TryGetColumnIndex("Output", out int output);
using (var cursor = result.GetRowCursor(result.Schema["Output"]))
{
var buffer = default(VBuffer <float>);
var getter = cursor.GetGetter <VBuffer <float> >(output);
var numRows = 0;
while (cursor.MoveNext())
{
getter(ref buffer);
Assert.Equal(10, buffer.Length);
numRows += 1;
}
Assert.Equal(4, numRows);
}
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only
public void TensorFlowTransformCifar()
{
var model_location = "cifar_model/frozen_model.pb";
var mlContext = new MLContext(seed: 1, conc: 1);
var tensorFlowModel = TensorFlowUtils.LoadTensorFlowModel(mlContext, model_location);
var schema = tensorFlowModel.GetInputSchema();
Assert.True(schema.TryGetColumnIndex("Input", out int column));
var type = (VectorType)schema.GetColumnType(column);
var imageHeight = type.Dimensions[0];
var imageWidth = type.Dimensions[1];
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = TextLoader.Create(mlContext, new TextLoader.Arguments()
{
Column = new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
}
}, new MultiFileSource(dataFile));
var pipeEstimator = new ImageLoadingEstimator(mlContext, imageFolder, ("ImagePath", "ImageReal"))
.Append(new ImageResizingEstimator(mlContext, "ImageReal", "ImageCropped", imageWidth, imageHeight))
.Append(new ImagePixelExtractingEstimator(mlContext, "ImageCropped", "Input", interleave: true));
var pixels = pipeEstimator.Fit(data).Transform(data);
IDataView trans = TensorFlowTransform.Create(mlContext, pixels, tensorFlowModel, new[] { "Output" }, new[] { "Input" });
trans.Schema.TryGetColumnIndex("Output", out int output);
using (var cursor = trans.GetRowCursor(col => col == output))
{
var buffer = default(VBuffer <float>);
var getter = cursor.GetGetter <VBuffer <float> >(output);
var numRows = 0;
while (cursor.MoveNext())
{
getter(ref buffer);
Assert.Equal(10, buffer.Length);
numRows += 1;
}
Assert.Equal(3, numRows);
}
}
public void TestTensorFlowStaticWithSchema()
{
var modelLocation = "cifar_model/frozen_model.pb";
using (var env = new ConsoleEnvironment())
{
var tensorFlowModel = TensorFlowUtils.LoadTensorFlowModel(env, modelLocation);
var schema = tensorFlowModel.GetInputSchema();
Assert.True(schema.TryGetColumnIndex("Input", out int column));
var type = schema.GetColumnType(column).AsVector;
var imageHeight = type.GetDim(0);
var imageWidth = type.GetDim(1);
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = TextLoader.CreateReader(env, ctx => (
imagePath: ctx.LoadText(0),
name: ctx.LoadText(1)))
.Read(new MultiFileSource(dataFile));
// Note that CamelCase column names are there to match the TF graph node names.
var pipe = data.MakeNewEstimator()
.Append(row => (
row.name,
Input: row.imagePath.LoadAsImage(imageFolder).Resize(imageHeight, imageWidth).ExtractPixels(interleaveArgb: true)))
.Append(row => (row.name, Output: row.Input.ApplyTensorFlowGraph(tensorFlowModel)));
TestEstimatorCore(pipe.AsDynamic, data.AsDynamic);
var result = pipe.Fit(data).Transform(data).AsDynamic;
result.Schema.TryGetColumnIndex("Output", out int output);
using (var cursor = result.GetRowCursor(col => col == output))
{
var buffer = default(VBuffer <float>);
var getter = cursor.GetGetter <VBuffer <float> >(output);
var numRows = 0;
while (cursor.MoveNext())
{
getter(ref buffer);
Assert.Equal(10, buffer.Length);
numRows += 1;
}
Assert.Equal(3, numRows);
}
}
}
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only
public void TensorFlowTransformCifarSavedModel()
{
var modelLocation = "cifar_saved_model";
var mlContext = new MLContext(seed: 1, conc: 1);
var loadModelSchema = TensorFlowUtils.GetModelSchema(mlContext, modelLocation);
Assert.Equal(335, loadModelSchema.Count);
var tensorFlowModel = TensorFlowUtils.LoadTensorFlowModel(mlContext, modelLocation);
var schema = tensorFlowModel.GetInputSchema();
Assert.True(schema.TryGetColumnIndex("Input", out int column));
var type = (VectorType)schema[column].Type;
var imageHeight = type.Dimensions[0];
var imageWidth = type.Dimensions[1];
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = mlContext.Data.ReadFromTextFile(dataFile, columns: new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
}
);
var images = new ImageLoaderTransformer(mlContext, imageFolder, ("ImagePath", "ImageReal")).Transform(data);
var cropped = new ImageResizerTransformer(mlContext, "ImageReal", "ImageCropped", imageWidth, imageHeight).Transform(images);
var pixels = new ImagePixelExtractorTransformer(mlContext, "ImageCropped", "Input", interleave: true).Transform(cropped);
IDataView trans = new TensorFlowTransformer(mlContext, tensorFlowModel, "Input", "Output").Transform(pixels);
trans.Schema.TryGetColumnIndex("Output", out int output);
using (var cursor = trans.GetRowCursor(col => col == output))
{
var buffer = default(VBuffer <float>);
var getter = cursor.GetGetter <VBuffer <float> >(output);
var numRows = 0;
while (cursor.MoveNext())
{
getter(ref buffer);
Assert.Equal(10, buffer.Length);
numRows += 1;
}
Assert.Equal(4, numRows);
}
}
/// <summary>
/// Example use of the TensorFlow sentiment classification model.
/// </summary>
public static void Example()
{
string modelLocation = SamplesUtils.DatasetUtils.DownloadTensorFlowSentimentModel();
var mlContext = new MLContext();
var data = new[] { new IMDBSentiment()
{
Sentiment_Text = "this film was just brilliant casting location scenery story direction " +
"everyone's really suited the part they played and you could just imagine being there robert " +
"is an amazing actor and now the same being director father came from the same scottish " +
"island as myself so i loved the fact there was a real connection with this film the witty " +
"remarks throughout the film were great it was just brilliant so much that i bought the " +
"film as soon as it was released for and would recommend it to everyone to watch and the " +
"fly fishing was amazing really cried at the end it was so sad and you know what they say " +
"if you cry at a film it must have been good and this definitely was also to the two " +
"little boy's that played the of norman and paul they were just brilliant children are " +
"often left out of the list i think because the stars that play them all grown up are " +
"such a big profile for the whole film but these children are amazing and should be praised " +
"for what they have done don't you think the whole story was so lovely because it was true " +
"and was someone's life after all that was shared with us all"
} };
var dataView = mlContext.Data.LoadFromEnumerable(data);
// This is the dictionary to convert words into the integer indexes.
var lookupMap = mlContext.Data.LoadFromTextFile(Path.Combine(modelLocation, "imdb_word_index.csv"),
columns: new[]
{
new TextLoader.Column("Words", DataKind.String, 0),
new TextLoader.Column("Ids", DataKind.Int32, 1),
},
separatorChar: ','
);
// Load the TensorFlow model once.
// - Use it for quering the schema for input and output in the model
// - Use it for prediction in the pipeline.
var modelInfo = TensorFlowUtils.LoadTensorFlowModel(mlContext, modelLocation);
var schema = modelInfo.GetModelSchema();
var featuresType = (VectorType)schema["Features"].Type;
Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Features", featuresType.ItemType.RawType, featuresType.Dimensions[0]);
var predictionType = (VectorType)schema["Prediction/Softmax"].Type;
Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Prediction/Softmax", predictionType.ItemType.RawType, predictionType.Dimensions[0]);
// The model expects the input feature vector to be a fixed length vector.
// In this sample, CustomMappingEstimator is used to resize variable length vector to fixed length vector.
// The following ML.NET pipeline
// 1. tokenzies the string into words,
// 2. maps each word to an integer which is an index in the dictionary ('lookupMap'),
// 3. Resizes the integer vector to a fixed length vector using CustomMappingEstimator ('ResizeFeaturesAction')
// 4. Passes the data to TensorFlow for scoring.
// 5. Retreives the 'Prediction' from TensorFlow and put it into ML.NET Pipeline
Action <IMDBSentiment, IntermediateFeatures> ResizeFeaturesAction = (i, j) =>
{
j.Sentiment_Text = i.Sentiment_Text;
var features = i.VariableLenghtFeatures;
Array.Resize(ref features, MaxSentenceLenth);
j.Features = features;
};
var engine = mlContext.Transforms.Text.TokenizeWords("TokenizedWords", "Sentiment_Text")
.Append(mlContext.Transforms.Conversion.ValueMap(lookupMap, "Words", "Ids", new ColumnOptions[] { ("VariableLenghtFeatures", "TokenizedWords") }))
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only
public void TensorFlowTransformCifarSavedModel()
{
var model_location = "cifar_saved_model";
var mlContext = new MLContext(seed: 1, conc: 1);
var tensorFlowModel = TensorFlowUtils.LoadTensorFlowModel(mlContext, model_location);
var schema = tensorFlowModel.GetInputSchema();
Assert.True(schema.TryGetColumnIndex("Input", out int column));
var type = (VectorType)schema.GetColumnType(column);
var imageHeight = type.Dimensions[0];
var imageWidth = type.Dimensions[1];
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = TextLoader.Create(mlContext, new TextLoader.Arguments()
{
Column = new[]
{
new TextLoader.Column("ImagePath", DataKind.TX, 0),
new TextLoader.Column("Name", DataKind.TX, 1),
}
}, new MultiFileSource(dataFile));
var images = ImageLoaderTransform.Create(mlContext, new ImageLoaderTransform.Arguments()
{
Column = new ImageLoaderTransform.Column[1]
{
new ImageLoaderTransform.Column()
{
Source = "ImagePath", Name = "ImageReal"
}
},
ImageFolder = imageFolder
}, data);
var cropped = ImageResizerTransform.Create(mlContext, new ImageResizerTransform.Arguments()
{
Column = new ImageResizerTransform.Column[1] {
new ImageResizerTransform.Column()
{
Source = "ImageReal", Name = "ImageCropped", ImageHeight = imageHeight, ImageWidth = imageWidth, Resizing = ImageResizerTransform.ResizingKind.IsoCrop
}
}
}, images);
var pixels = ImagePixelExtractorTransform.Create(mlContext, new ImagePixelExtractorTransform.Arguments()
{
Column = new ImagePixelExtractorTransform.Column[1] {
new ImagePixelExtractorTransform.Column()
{
Source = "ImageCropped", Name = "Input", UseAlpha = false, InterleaveArgb = true
}
}
}, cropped);
IDataView trans = TensorFlowTransform.Create(mlContext, pixels, tensorFlowModel, new[] { "Output" }, new[] { "Input" });
trans.Schema.TryGetColumnIndex("Output", out int output);
using (var cursor = trans.GetRowCursor(col => col == output))
{
var buffer = default(VBuffer <float>);
var getter = cursor.GetGetter <VBuffer <float> >(output);
var numRows = 0;
while (cursor.MoveNext())
{
getter(ref buffer);
Assert.Equal(10, buffer.Length);
numRows += 1;
}
Assert.Equal(3, numRows);
}
}
/// <summary> /// Load TensorFlow model into memory. This is the convenience method that allows the model to be loaded once and subsequently use it for querying schema and creation of /// <see cref="TensorFlowEstimator"/> using <see cref="TensorFlowModel.ScoreTensorFlowModel(string, string, bool)"/>. /// </summary> /// <param name="catalog">The transform's catalog.</param> /// <param name="modelLocation">Location of the TensorFlow model.</param> /// <example> /// <format type="text/markdown"> /// <] /// ]]> /// </format> /// </example> public static TensorFlowModel LoadTensorFlowModel(this ModelOperationsCatalog catalog, string modelLocation) => TensorFlowUtils.LoadTensorFlowModel(CatalogUtils.GetEnvironment(catalog), modelLocation);
/// <summary> /// Load TensorFlow model into memory. This is the convenience method that allows the model to be loaded once and subsequently use it for querying schema and creation of /// <see cref="TensorFlowEstimator"/> using <see cref="TensorFlowModel.ScoreTensorFlowModel(string, string, bool)"/>. /// usage of this API requires additional NuGet dependencies on TensorFlow redist, see linked document for more information. /// <see cref="TensorFlowModel"/> also holds references to unmanaged resources that need to be freed either with an explicit /// call to Dispose() or implicitly by declaring the variable with the "using" syntax/> /// /// <format type="text/markdown"> /// <] /// ]]> /// </format> /// </summary> /// <param name="catalog">The transform's catalog.</param> /// <param name="modelLocation">Location of the TensorFlow model.</param> /// <param name="treatOutputAsBatched">If the first dimension of the output is unknown, should it be treated as batched or not.</param> /// <example> /// <format type="text/markdown"> /// <] /// ]]> /// </format> /// </example> public static TensorFlowModel LoadTensorFlowModel(this ModelOperationsCatalog catalog, string modelLocation, bool treatOutputAsBatched) => TensorFlowUtils.LoadTensorFlowModel(CatalogUtils.GetEnvironment(catalog), modelLocation, treatOutputAsBatched);
