void FileBasedSavingOfData()
{
var dataPath = GetDataPath(SentimentDataPath);
var testDataPath = GetDataPath(SentimentTestPath);
using (var env = new TlcEnvironment(seed: 1, conc: 1))
{
// Pipeline
var loader = new TextLoader(env, MakeSentimentTextLoaderArgs(), new MultiFileSource(dataPath));
var trans = TextTransform.Create(env, MakeSentimentTextTransformArgs(), loader);
var saver = new BinarySaver(env, new BinarySaver.Arguments());
using (var ch = env.Start("SaveData"))
using (var file = env.CreateOutputFile("i.idv"))
{
DataSaverUtils.SaveDataView(ch, saver, trans, file);
}
var binData = new BinaryLoader(env, new BinaryLoader.Arguments(), new MultiFileSource("i.idv"));
var trainRoles = new RoleMappedData(binData, label: "Label", feature: "Features");
var trainer = new LinearClassificationTrainer(env, new LinearClassificationTrainer.Arguments
{
NumThreads = 1
});
var predictor = trainer.Train(new Runtime.TrainContext(trainRoles));
DeleteOutputPath("i.idv");
}
}
void New_FileBasedSavingOfData()
{
var dataPath = GetDataPath(SentimentDataPath);
var testDataPath = GetDataPath(SentimentTestPath);
using (var env = new TlcEnvironment(seed: 1, conc: 1))
{
// Pipeline.
var pipeline = new MyTextLoader(env, MakeSentimentTextLoaderArgs())
.Append(new MyTextTransform(env, MakeSentimentTextTransformArgs()));
var trainData = pipeline.Fit(new MultiFileSource(dataPath)).Read(new MultiFileSource(dataPath));
using (var file = env.CreateOutputFile("i.idv"))
trainData.SaveAsBinary(env, file.CreateWriteStream());
var trainer = new MySdca(env, new LinearClassificationTrainer.Arguments {
NumThreads = 1
}, "Features", "Label");
var loadedTrainData = new BinaryLoader(env, new BinaryLoader.Arguments(), new MultiFileSource("i.idv"));
// Train.
var model = trainer.Train(loadedTrainData);
DeleteOutputPath("i.idv");
}
}
void New_FileBasedSavingOfData()
{
var dataPath = GetDataPath(SentimentDataPath);
var testDataPath = GetDataPath(SentimentTestPath);
using (var env = new TlcEnvironment(seed: 1, conc: 1))
{
var trainData = new TextLoader(env, MakeSentimentTextLoaderArgs())
.Append(new TextTransform(env, "SentimentText", "Features"))
.FitAndRead(new MultiFileSource(dataPath));
using (var file = env.CreateOutputFile("i.idv"))
trainData.SaveAsBinary(env, file.CreateWriteStream());
var trainer = new LinearClassificationTrainer(env, new LinearClassificationTrainer.Arguments {
NumThreads = 1
}, "Features", "Label");
var loadedTrainData = new BinaryLoader(env, new BinaryLoader.Arguments(), new MultiFileSource("i.idv"));
// Train.
var model = trainer.Train(new RoleMappedData(loadedTrainData, DefaultColumnNames.Label, DefaultColumnNames.Features));
DeleteOutputPath("i.idv");
}
}
public void TestBackAndForthConversion()
{
using (var env = new TlcEnvironment())
{
var imageHeight = 100;
var imageWidth = 130;
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
{
Column = new ImageLoaderTransform.Column[1]
{
new ImageLoaderTransform.Column()
{
Source = "ImagePath", Name = "ImageReal"
}
},
ImageFolder = imageFolder
}, data);
var cropped = ImageResizerTransform.Create(env, new ImageResizerTransform.Arguments()
{
Column = new ImageResizerTransform.Column[1] {
new ImageResizerTransform.Column()
{
Source = "ImageReal", Name = "ImageCropped", ImageHeight = imageHeight, ImageWidth = imageWidth, Resizing = ImageResizerTransform.ResizingKind.IsoCrop
}
}
}, images);
var pixels = ImagePixelExtractorTransform.Create(env, new ImagePixelExtractorTransform.Arguments()
{
Column = new ImagePixelExtractorTransform.Column[1] {
new ImagePixelExtractorTransform.Column()
{
Source = "ImageCropped", Name = "ImagePixels", UseAlpha = true
}
}
}, cropped);
IDataView backToBitmaps = new VectorToImageTransform(env, new VectorToImageTransform.Arguments()
{
Column = new VectorToImageTransform.Column[1] {
new VectorToImageTransform.Column()
{
Source = "ImagePixels", Name = "ImageRestored", ImageHeight = imageHeight, ImageWidth = imageWidth, ContainsAlpha = true
}
}
}, pixels);
var fname = nameof(TestBackAndForthConversion) + "_model.zip";
var fh = env.CreateOutputFile(fname);
using (var ch = env.Start("save"))
TrainUtils.SaveModel(env, ch, fh, null, new RoleMappedData(backToBitmaps));
backToBitmaps = ModelFileUtils.LoadPipeline(env, fh.OpenReadStream(), new MultiFileSource(dataFile));
DeleteOutputPath(fname);
backToBitmaps.Schema.TryGetColumnIndex("ImageRestored", out int bitmapColumn);
backToBitmaps.Schema.TryGetColumnIndex("ImageCropped", out int cropBitmapColumn);
using (var cursor = backToBitmaps.GetRowCursor((x) => true))
{
var bitmapGetter = cursor.GetGetter <Bitmap>(bitmapColumn);
Bitmap restoredBitmap = default;
var bitmapCropGetter = cursor.GetGetter <Bitmap>(cropBitmapColumn);
Bitmap croppedBitmap = default;
while (cursor.MoveNext())
{
bitmapGetter(ref restoredBitmap);
Assert.NotNull(restoredBitmap);
bitmapCropGetter(ref croppedBitmap);
Assert.NotNull(croppedBitmap);
for (int x = 0; x < imageWidth; x++)
{
for (int y = 0; y < imageHeight; y++)
{
Assert.True(croppedBitmap.GetPixel(x, y) == restoredBitmap.GetPixel(x, y));
}
}
}
}
}
Done();
}
public void TestGreyscaleTransformImages()
{
using (var env = new TlcEnvironment())
{
var imageHeight = 150;
var imageWidth = 100;
var dataFile = GetDataPath("images/images.tsv");
var imageFolder = Path.GetDirectoryName(dataFile);
var data = env.CreateLoader("Text{col=ImagePath:TX:0 col=Name:TX:1}", new MultiFileSource(dataFile));
var images = ImageLoaderTransform.Create(env, new ImageLoaderTransform.Arguments()
{
Column = new ImageLoaderTransform.Column[1]
{
new ImageLoaderTransform.Column()
{
Source = "ImagePath", Name = "ImageReal"
}
},
ImageFolder = imageFolder
}, data);
var cropped = ImageResizerTransform.Create(env, new ImageResizerTransform.Arguments()
{
Column = new ImageResizerTransform.Column[1] {
new ImageResizerTransform.Column()
{
Name = "ImageCropped", Source = "ImageReal", ImageHeight = imageHeight, ImageWidth = imageWidth, Resizing = ImageResizerTransform.ResizingKind.IsoCrop
}
}
}, images);
IDataView grey = ImageGrayscaleTransform.Create(env, new ImageGrayscaleTransform.Arguments()
{
Column = new ImageGrayscaleTransform.Column[1] {
new ImageGrayscaleTransform.Column()
{
Name = "ImageGrey", Source = "ImageCropped"
}
}
}, cropped);
var fname = nameof(TestGreyscaleTransformImages) + "_model.zip";
var fh = env.CreateOutputFile(fname);
using (var ch = env.Start("save"))
TrainUtils.SaveModel(env, ch, fh, null, new RoleMappedData(grey));
grey = ModelFileUtils.LoadPipeline(env, fh.OpenReadStream(), new MultiFileSource(dataFile));
DeleteOutputPath(fname);
grey.Schema.TryGetColumnIndex("ImageGrey", out int greyColumn);
using (var cursor = grey.GetRowCursor((x) => true))
{
var bitmapGetter = cursor.GetGetter <Bitmap>(greyColumn);
Bitmap bitmap = default;
while (cursor.MoveNext())
{
bitmapGetter(ref bitmap);
Assert.NotNull(bitmap);
for (int x = 0; x < imageWidth; x++)
{
for (int y = 0; y < imageHeight; y++)
{
var pixel = bitmap.GetPixel(x, y);
// greyscale image has same values for R,G and B
Assert.True(pixel.R == pixel.G && pixel.G == pixel.B);
}
}
}
}
}
Done();
}
