public void CategoricalHashStatic()
{
string dataPath = GetDataPath("breast-cancer.txt");
var reader = TextLoader.CreateReader(Env, ctx => (
ScalarString: ctx.LoadText(1),
VectorString: ctx.LoadText(1, 4)));
var data = reader.Read(dataPath);
var wrongCollection = new[] { new TestClass()
{
A = "1", B = "2", C = "3",
}, new TestClass()
{
A = "4", B = "5", C = "6"
} };
var invalidData = ComponentCreation.CreateDataView(Env, wrongCollection);
var est = data.MakeNewEstimator().
Append(row => (
row.ScalarString,
row.VectorString,
// Create a VarVector column
VarVectorString: row.ScalarString.TokenizeText())).
Append(row => (
A: row.ScalarString.OneHotHashEncoding(outputKind: CategoricalHashStaticExtensions.OneHotHashScalarOutputKind.Ind),
B: row.VectorString.OneHotHashEncoding(outputKind: CategoricalHashStaticExtensions.OneHotHashVectorOutputKind.Ind),
C: row.VectorString.OneHotHashEncoding(outputKind: CategoricalHashStaticExtensions.OneHotHashVectorOutputKind.Bag),
D: row.ScalarString.OneHotHashEncoding(outputKind: CategoricalHashStaticExtensions.OneHotHashScalarOutputKind.Bin),
E: row.VectorString.OneHotHashEncoding(outputKind: CategoricalHashStaticExtensions.OneHotHashVectorOutputKind.Bin),
F: row.VarVectorString.OneHotHashEncoding()
));
TestEstimatorCore(est.AsDynamic, data.AsDynamic, invalidInput: invalidData);
var outputPath = GetOutputPath("CategoricalHash", "featurized.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
var savedData = TakeFilter.Create(Env, est.Fit(data).Transform(data).AsDynamic, 4);
var view = ColumnSelectingTransformer.CreateKeep(Env, savedData, new[] { "A", "B", "C", "D", "E", "F" });
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, view, fs, keepHidden: true);
}
CheckEquality("CategoricalHash", "featurized.tsv");
Done();
}
public void LdaWorkout()
{
IHostEnvironment env = new MLContext(seed: 42, conc: 1);
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoaderStatic.CreateReader(env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(sentimentDataPath);
var invalidData = TextLoaderStatic.CreateReader(env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(sentimentDataPath);
var est = new WordBagEstimator(env, "text", "bag_of_words").
Append(new LatentDirichletAllocationEstimator(env, "bag_of_words", "topics", 10, numIterations: 10,
resetRandomGenerator: true));
// The following call fails because of the following issue
// https://github.com/dotnet/machinelearning/issues/969
// In this test it manifests because of the WordBagEstimator in the estimator chain
// TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "ldatopics.tsv");
using (var ch = env.Start("save"))
{
var saver = new TextSaver(env, new TextSaver.Arguments {
Silent = true, OutputHeader = false, Dense = true
});
var transformer = est.Fit(data.AsDynamic);
var transformedData = transformer.Transform(data.AsDynamic);
IDataView savedData = TakeFilter.Create(env, transformedData, 4);
savedData = ColumnSelectingTransformer.CreateKeep(env, savedData, new[] { "topics" });
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
Assert.Equal(10, (savedData.Schema[0].Type as VectorType)?.Size);
}
// Diabling this check due to the following issue with consitency of output.
// `seed` specified in ConsoleEnvironment has no effect.
// https://github.com/dotnet/machinelearning/issues/1004
// On single box, setting `s.ResetRandomGenerator = true` works but fails on build server
// CheckEquality("Text", "ldatopics.tsv");
Done();
}
public void LdaWorkout()
{
var env = new ConsoleEnvironment(seed: 42, conc: 1);
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoader.CreateReader(env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(new MultiFileSource(sentimentDataPath));
var invalidData = TextLoader.CreateReader(env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(new MultiFileSource(sentimentDataPath));
var est = new WordBagEstimator(env, "text", "bag_of_words").
Append(new LdaEstimator(env, "bag_of_words", "topics", 10, advancedSettings: s => {
s.NumIterations = 10;
s.ResetRandomGenerator = true;
}));
// The following call fails because of the following issue
// https://github.com/dotnet/machinelearning/issues/969
// TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "ldatopics.tsv");
using (var ch = env.Start("save"))
{
var saver = new TextSaver(env, new TextSaver.Arguments {
Silent = true, OutputHeader = false, Dense = true
});
IDataView savedData = TakeFilter.Create(env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(env, savedData, "topics");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
Assert.Equal(10, savedData.Schema.GetColumnType(0).VectorSize);
}
// Diabling this check due to the following issue with consitency of output.
// `seed` specified in ConsoleEnvironment has no effect.
// https://github.com/dotnet/machinelearning/issues/1004
// On single box, setting `s.ResetRandomGenerator = true` works but fails on build server
// CheckEquality("Text", "ldatopics.tsv");
Done();
}
public void NAReplaceStatic()
{
string dataPath = GetDataPath("breast-cancer.txt");
var reader = TextLoader.CreateReader(Env, ctx => (
ScalarFloat: ctx.LoadFloat(1),
ScalarDouble: ctx.LoadDouble(1),
VectorFloat: ctx.LoadFloat(1, 4),
VectorDoulbe: ctx.LoadDouble(1, 4)
));
var data = reader.Read(new MultiFileSource(dataPath));
var wrongCollection = new[] { new TestClass()
{
A = 1, B = 3, C = new float[2] {
1, 2
}, D = new double[2] {
3, 4
}
} };
var invalidData = ComponentCreation.CreateDataView(Env, wrongCollection);
var est = data.MakeNewEstimator().
Append(row => (
A: row.ScalarFloat.ReplaceWithMissingValues(NAReplaceTransform.ColumnInfo.ReplacementMode.Maximum),
B: row.ScalarDouble.ReplaceWithMissingValues(NAReplaceTransform.ColumnInfo.ReplacementMode.Mean),
C: row.VectorFloat.ReplaceWithMissingValues(NAReplaceTransform.ColumnInfo.ReplacementMode.Mean),
D: row.VectorDoulbe.ReplaceWithMissingValues(NAReplaceTransform.ColumnInfo.ReplacementMode.Minimum)
));
TestEstimatorCore(est.AsDynamic, data.AsDynamic, invalidInput: invalidData);
var outputPath = GetOutputPath("NAReplace", "featurized.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data).Transform(data).AsDynamic, 4);
savedData = new ChooseColumnsTransform(Env, savedData, "A", "B", "C", "D");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("NAReplace", "featurized.tsv");
Done();
}
void TestDifferentTypes()
{
string dataPath = GetDataPath("adult.test");
var loader = new TextLoader(Env, new TextLoader.Arguments
{
Column = new[] {
new TextLoader.Column("float1", DataKind.R4, 0),
new TextLoader.Column("float4", DataKind.R4, new[] { new TextLoader.Range(0), new TextLoader.Range(2), new TextLoader.Range(4), new TextLoader.Range(10) }),
new TextLoader.Column("double1", DataKind.R8, 0),
new TextLoader.Column("double4", DataKind.R8, new[] { new TextLoader.Range(0), new TextLoader.Range(2), new TextLoader.Range(4), new TextLoader.Range(10) }),
new TextLoader.Column("int1", DataKind.I4, 0),
new TextLoader.Column("text1", DataKind.TX, 1),
new TextLoader.Column("text2", DataKind.TX, new[] { new TextLoader.Range(1), new TextLoader.Range(3) }),
},
Separator = ",",
HasHeader = true
}, new MultiFileSource(dataPath));
var pipe = new ValueToKeyMappingEstimator(Env, new[] {
new ValueToKeyMappingTransformer.ColumnInfo("float1", "TermFloat1"),
new ValueToKeyMappingTransformer.ColumnInfo("float4", "TermFloat4"),
new ValueToKeyMappingTransformer.ColumnInfo("double1", "TermDouble1"),
new ValueToKeyMappingTransformer.ColumnInfo("double4", "TermDouble4"),
new ValueToKeyMappingTransformer.ColumnInfo("int1", "TermInt1"),
new ValueToKeyMappingTransformer.ColumnInfo("text1", "TermText1"),
new ValueToKeyMappingTransformer.ColumnInfo("text2", "TermText2")
});
var data = loader.Read(dataPath);
data = TakeFilter.Create(Env, data, 10);
var outputPath = GetOutputPath("Term", "Term.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, pipe.Fit(data).Transform(data), fs, keepHidden: true);
}
CheckEquality("Term", "Term.tsv");
Done();
}
public void CategoricalStatic()
{
string dataPath = GetDataPath("breast-cancer.txt");
var reader = TextLoaderStatic.CreateReader(Env, ctx => (
ScalarString: ctx.LoadText(1),
VectorString: ctx.LoadText(1, 4)));
var data = reader.Read(dataPath);
var wrongCollection = new[] { new TestClass()
{
A = 1, B = 2, C = 3,
}, new TestClass()
{
A = 4, B = 5, C = 6
} };
var invalidData = ML.Data.ReadFromEnumerable(wrongCollection);
var est = data.MakeNewEstimator().
Append(row => (
A: row.ScalarString.OneHotEncoding(outputKind: CategoricalStaticExtensions.OneHotScalarOutputKind.Ind),
B: row.VectorString.OneHotEncoding(outputKind: CategoricalStaticExtensions.OneHotVectorOutputKind.Ind),
C: row.VectorString.OneHotEncoding(outputKind: CategoricalStaticExtensions.OneHotVectorOutputKind.Bag),
D: row.ScalarString.OneHotEncoding(outputKind: CategoricalStaticExtensions.OneHotScalarOutputKind.Bin),
E: row.VectorString.OneHotEncoding(outputKind: CategoricalStaticExtensions.OneHotVectorOutputKind.Bin)
));
TestEstimatorCore(est.AsDynamic, data.AsDynamic, invalidInput: invalidData);
var outputPath = GetOutputPath("Categorical", "featurized.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
var savedData = TakeFilter.Create(Env, est.Fit(data).Transform(data).AsDynamic, 4);
var view = new ColumnSelectingTransformer(Env, new string[] { "A", "B", "C", "D", "E" }, null, false).Transform(savedData);
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, view, fs, keepHidden: true);
}
CheckEquality("Categorical", "featurized.tsv");
Done();
}
public void NormalizerWithOnFit()
{
var env = new ConsoleEnvironment(seed: 0);
var dataPath = GetDataPath("generated_regression_dataset.csv");
var dataSource = new MultiFileSource(dataPath);
var reader = TextLoader.CreateReader(env,
c => c.LoadFloat(0, 2),
separator: ';', hasHeader: true);
var data = reader.Read(dataSource);
// These will be populated once we call fit.
ImmutableArray <float> mm;
ImmutableArray <float> ss;
ImmutableArray <ImmutableArray <float> > bb;
var est = reader.MakeNewEstimator()
.Append(r => (r,
ncdf: r.NormalizeByCumulativeDistribution(onFit: (m, s) => mm = m),
n: r.NormalizeByMeanVar(onFit: (s, o) => { ss = s; Assert.Empty(o); }),
b: r.NormalizeByBinning(onFit: b => bb = b)));
var tdata = est.Fit(data).Transform(data);
Assert.Equal(3, mm.Length);
Assert.Equal(3, ss.Length);
Assert.Equal(3, bb.Length);
// Just for fun, let's also write out some of the lines of the data to the console.
using (var stream = new MemoryStream())
{
IDataView v = new ChooseColumnsTransform(env, tdata.AsDynamic, "r", "ncdf", "n", "b");
v = TakeFilter.Create(env, v, 10);
var saver = new TextSaver(env, new TextSaver.Arguments()
{
Dense = true,
Separator = ",",
OutputHeader = false
});
saver.SaveData(stream, v, Utils.GetIdentityPermutation(v.Schema.ColumnCount));
Console.WriteLine(Encoding.UTF8.GetString(stream.ToArray()));
}
}
public void TestWordEmbeddings()
{
var dataPath = GetDataPath(TestDatasets.Sentiment.trainFilename);
var data = new TextLoader(Env,
new TextLoader.Arguments()
{
Separator = "\t",
HasHeader = true,
Columns = new[]
{
new TextLoader.Column("Label", DataKind.BL, 0),
new TextLoader.Column("SentimentText", DataKind.Text, 1)
}
}).Read(GetDataPath(dataPath));
var est = ML.Transforms.Text.NormalizeText("NormalizedText", "SentimentText", keepDiacritics: false, keepPunctuations: false)
.Append(ML.Transforms.Text.TokenizeWords("Words", "NormalizedText"))
.Append(ML.Transforms.Text.RemoveDefaultStopWords("CleanWords", "Words"));
var words = est.Fit(data).Transform(data);
var pipe = ML.Transforms.Text.ExtractWordEmbeddings("WordEmbeddings", "CleanWords", modelKind: WordEmbeddingsExtractingTransformer.PretrainedModelKind.Sswe);
TestEstimatorCore(pipe, words, invalidInput: data);
var outputPath = GetOutputPath("Text", "wordEmbeddings.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, pipe.Fit(words).Transform(words), 4);
savedData = ColumnSelectingTransformer.CreateKeep(Env, savedData, new[] { "WordEmbeddings" });
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "wordEmbeddings.tsv");
Done();
}
public void NgramWorkout()
{
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(new MultiFileSource(sentimentDataPath));
var invalidData = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(new MultiFileSource(sentimentDataPath));
var est = new WordTokenizer(Env, "text", "text")
.Append(new TermEstimator(Env, "text", "terms"))
.Append(new NgramEstimator(Env, "terms", "ngrams"))
.Append(new NgramHashEstimator(Env, "terms", "ngramshash"));
// The following call fails because of the following issue
// https://github.com/dotnet/machinelearning/issues/969
// TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "ngrams.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(Env, savedData, "text", "terms", "ngrams", "ngramshash");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "ngrams.tsv");
Done();
}
public void PcaWorkout()
{
var env = new ConsoleEnvironment(seed: 1, conc: 1);
string dataSource = GetDataPath("generated_regression_dataset.csv");
var data = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadFloat(0, 10)),
separator: ';', hasHeader: true)
.Read(new MultiFileSource(dataSource));
var invalidData = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadText(0, 10)),
separator: ';', hasHeader: true)
.Read(new MultiFileSource(dataSource));
var est = new PcaEstimator(env, "features", "pca", rank: 5, advancedSettings: s => {
s.Seed = 1;
});
// The following call fails because of the following issue
// https://github.com/dotnet/machinelearning/issues/969
// TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("PCA", "pca.tsv");
using (var ch = env.Start("save"))
{
var saver = new TextSaver(env, new TextSaver.Arguments {
Silent = true, OutputHeader = false
});
IDataView savedData = TakeFilter.Create(env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(env, savedData, "pca");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("PCA", "pca.tsv");
Done();
}
public void TestPcaEstimator()
{
var data = TextLoader.CreateReader(_env,
c => (label: c.LoadFloat(11), features: c.LoadFloat(0, 10)),
separator: ';', hasHeader: true)
.Read(_dataSource);
var est = new PcaEstimator(_env, "features", "pca", rank: 5, seed: 1);
var outputPath = GetOutputPath("PCA", "pca.tsv");
using (var ch = _env.Start("save"))
{
IDataView savedData = TakeFilter.Create(_env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(_env, savedData, "pca");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, _saver, savedData, fs, keepHidden: true);
}
CheckEquality("PCA", "pca.tsv", digitsOfPrecision: 4);
Done();
}
public void NgramWorkout()
{
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoaderStatic.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(sentimentDataPath);
var invalidData = TextLoaderStatic.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(sentimentDataPath);
var est = new WordTokenizingEstimator(Env, "text", "text")
.Append(new ValueToKeyMappingEstimator(Env, "text", "terms"))
.Append(new NgramExtractingEstimator(Env, "terms", "ngrams"))
.Append(new NgramHashingEstimator(Env, "terms", "ngramshash"));
TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "ngrams.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = ColumnSelectingTransformer.CreateKeep(Env, savedData, new[] { "text", "terms", "ngrams", "ngramshash" });
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "ngrams.tsv");
Done();
}
public void WordBagWorkout()
{
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoaderStatic.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(sentimentDataPath);
var invalidData = TextLoaderStatic.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(sentimentDataPath);
var est = new WordBagEstimator(Env, "text", "bag_of_words").
Append(new WordHashBagEstimator(Env, "text", "bag_of_wordshash", invertHash: -1));
// The following call fails because of the following issue
// https://github.com/dotnet/machinelearning/issues/969
// TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "bag_of_words.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = ColumnSelectingTransformer.CreateKeep(Env, savedData, new[] { "text", "bag_of_words", "bag_of_wordshash" });
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "bag_of_words.tsv");
Done();
}
public void LpGcNormAndWhiteningWorkout()
{
var env = new ConsoleEnvironment(seed: 0);
string dataSource = GetDataPath("generated_regression_dataset.csv");
var data = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadFloat(0, 10)),
separator: ';', hasHeader: true)
.Read(new MultiFileSource(dataSource));
var invalidData = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadText(0, 10)),
separator: ';', hasHeader: true)
.Read(new MultiFileSource(dataSource));
var est = new LpNormalizer(env, "features", "lpnorm")
.Append(new GlobalContrastNormalizer(env, "features", "gcnorm"))
.Append(new Whitening(env, "features", "whitened"));
TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "lpnorm_gcnorm_whitened.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true, OutputHeader = false
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(Env, savedData, "lpnorm", "gcnorm", "whitened");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "lpnorm_gcnorm_whitened.tsv", digitsOfPrecision: 4);
Done();
}
public void TextFeaturizerWorkout()
{
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(new MultiFileSource(sentimentDataPath));
var invalidData = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(new MultiFileSource(sentimentDataPath))
.AsDynamic;
//var feat = Estimator.MakeNew(data)
// .Append(row => row.text.FeaturizeText(advancedSettings: s => { s.OutputTokens = true; }));
var feat = new TextTransform(Env, "text", "Data", advancedSettings: s => { s.OutputTokens = true; });
TestEstimatorCore(feat, data.AsDynamic, invalidInput: invalidData);
var outputPath = GetOutputPath("Text", "featurized.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, feat.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(Env, savedData, "Data", "Data_TransformedText");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "featurized.tsv");
Done();
}
private void TestFeatureContribution(
ITrainerEstimator <ISingleFeaturePredictionTransformer <IPredictor>, IPredictor> trainer,
IDataView data,
string testFile,
int precision = 6)
{
// Train the model.
var model = trainer.Fit(data);
// Extract the predictor, check that it supports feature contribution.
var predictor = model.Model as ICalculateFeatureContribution;
Assert.NotNull(predictor);
// Calculate feature contributions.
var est = new FeatureContributionCalculatingEstimator(ML, predictor, "Features", numPositiveContributions: 3, numNegativeContributions: 0)
.Append(new FeatureContributionCalculatingEstimator(ML, predictor, "Features", numPositiveContributions: 0, numNegativeContributions: 3))
.Append(new FeatureContributionCalculatingEstimator(ML, predictor, "Features", numPositiveContributions: 1, numNegativeContributions: 1))
.Append(new FeatureContributionCalculatingEstimator(ML, predictor, "Features", numPositiveContributions: 1, numNegativeContributions: 1, normalize: false));
TestEstimatorCore(est, data);
// Verify output.
var outputPath = GetOutputPath("FeatureContribution", testFile + ".tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(ML, new TextSaver.Arguments {
Silent = true, OutputHeader = false
});
IDataView savedData = TakeFilter.Create(ML, est.Fit(data).Transform(data), 4);
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("FeatureContribution", testFile + ".tsv", digitsOfPrecision: precision);
Done();
}
public void TextNormalizationAndStopwordRemoverWorkout()
{
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoaderStatic.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(sentimentDataPath);
var invalidData = TextLoaderStatic.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(sentimentDataPath);
var est = ML.Transforms.Text.NormalizeText("text")
.Append(ML.Transforms.Text.TokenizeWords("text", "words"))
.Append(ML.Transforms.Text.RemoveDefaultStopWords("words", "NoDefaultStopwords"))
.Append(ML.Transforms.Text.RemoveStopWords("words", "NoStopWords", "xbox", "this", "is", "a", "the", "THAT", "bY"));
TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "words_without_stopwords.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = ColumnSelectingTransformer.CreateKeep(Env, savedData, new[] { "text", "NoDefaultStopwords", "NoStopWords" });
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "words_without_stopwords.tsv");
Done();
}
public void TextNormalizationAndStopwordRemoverWorkout()
{
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(new MultiFileSource(sentimentDataPath));
var invalidData = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(new MultiFileSource(sentimentDataPath));
var est = new TextNormalizer(Env, "text")
.Append(new WordTokenizer(Env, "text", "words"))
.Append(new StopwordRemover(Env, "words", "words_without_stopwords"));
TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "words_without_stopwords.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(Env, savedData, "text", "words_without_stopwords");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "words_without_stopwords.tsv");
Done();
}
public void WhiteningWorkout()
{
var env = new ConsoleEnvironment(seed: 0);
string dataSource = GetDataPath("generated_regression_dataset.csv");
var data = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadFloat(0, 10)),
separator: ';', hasHeader: true)
.Read(dataSource);
var invalidData = TextLoader.CreateReader(env,
c => (label: c.LoadFloat(11), features: c.LoadText(0, 10)),
separator: ';', hasHeader: true)
.Read(dataSource);
var est = new VectorWhiteningEstimator(env, "features", "whitened1")
.Append(new VectorWhiteningEstimator(env, "features", "whitened2", kind: WhiteningKind.Pca, pcaNum: 5));
TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("NormalizerEstimator", "whitened.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true, OutputHeader = false
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = SelectColumnsTransform.CreateKeep(Env, savedData, new[] { "whitened1", "whitened2" });
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("NormalizerEstimator", "whitened.tsv", digitsOfPrecision: 4);
Done();
}
public void LpGcNormAndWhiteningWorkout()
{
string dataSource = GetDataPath(TestDatasets.generatedRegressionDataset.trainFilename);
var data = TextLoader.CreateReader(ML,
c => (label: c.LoadFloat(11), features: c.LoadFloat(0, 10)),
separator: ';', hasHeader: true)
.Read(dataSource);
var invalidData = TextLoader.CreateReader(ML,
c => (label: c.LoadFloat(11), features: c.LoadText(0, 10)),
separator: ';', hasHeader: true)
.Read(dataSource);
var est = new LpNormalizingEstimator(ML, "features", "lpnorm")
.Append(new GlobalContrastNormalizingEstimator(ML, "features", "gcnorm"))
.Append(new VectorWhiteningEstimator(ML, "features", "whitened"));
TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("NormalizerEstimator", "lpnorm_gcnorm_whitened.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(ML, new TextSaver.Arguments {
Silent = true, OutputHeader = false
});
IDataView savedData = TakeFilter.Create(ML, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = ColumnSelectingTransformer.CreateKeep(ML, savedData, new[] { "lpnorm", "gcnorm", "whitened" });
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("NormalizerEstimator", "lpnorm_gcnorm_whitened.tsv", digitsOfPrecision: 4);
Done();
}
public void TestConvertWorkout()
{
var data = new[] { new TestClass()
{
A = 1, B = new int[2] {
1, 4
}
},
new TestClass()
{
A = 2, B = new int[2] {
3, 4
}
} };
var dataView = ComponentCreation.CreateDataView(Env, data);
var pipe = new ConvertingEstimator(Env, columns: new[] { new ConvertingTransform.ColumnInfo("A", "ConvA", DataKind.R4),
new ConvertingTransform.ColumnInfo("B", "ConvB", DataKind.R4) });
TestEstimatorCore(pipe, dataView);
var allTypesData = new[]
{
new TestPrimitiveClass()
{
AA = new [] { "a", "b" },
AB = new [] { false, true },
AC = new [] { -1, 1 },
AD = new uint[] { 0, 1 },
AE = new byte[] { 0, 1 },
AF = new sbyte[] { -1, 1 },
AG = new short[] { -1, 1 },
AH = new ushort[] { 0, 1 },
AK = new long[] { -1, 1 },
AL = new ulong[] { 0, 1 },
AM = new float[] { 1.0f, 1.0f, },
AN = new double[] { 1.0d, 1.0d, }
},
new TestPrimitiveClass()
{
AA = new [] { "0", "1" },
AB = new [] { false, true },
AC = new [] { int.MinValue, int.MaxValue },
AD = new uint[] { uint.MinValue, uint.MaxValue },
AE = new byte[] { byte.MinValue, byte.MaxValue },
AF = new sbyte[] { sbyte.MinValue, sbyte.MaxValue },
AG = new short[] { short.MinValue, short.MaxValue },
AH = new ushort[] { ushort.MinValue, ushort.MaxValue },
AK = new long[] { long.MinValue, long.MaxValue },
AL = new ulong[] { ulong.MinValue, ulong.MaxValue },
AM = new float[] { float.MinValue, float.MaxValue, },
AN = new double[] { double.MinValue, double.MaxValue, }
}
};
var allTypesDataView = ComponentCreation.CreateDataView(Env, allTypesData);
var allTypesPipe = new ConvertingEstimator(Env, columns: new[] {
new ConvertingTransform.ColumnInfo("AA", "ConvA", DataKind.R4),
new ConvertingTransform.ColumnInfo("AB", "ConvB", DataKind.R4),
new ConvertingTransform.ColumnInfo("AC", "ConvC", DataKind.R4),
new ConvertingTransform.ColumnInfo("AD", "ConvD", DataKind.R4),
new ConvertingTransform.ColumnInfo("AE", "ConvE", DataKind.R4),
new ConvertingTransform.ColumnInfo("AF", "ConvF", DataKind.R4),
new ConvertingTransform.ColumnInfo("AG", "ConvG", DataKind.R4),
new ConvertingTransform.ColumnInfo("AH", "ConvH", DataKind.R4),
new ConvertingTransform.ColumnInfo("AK", "ConvK", DataKind.R4),
new ConvertingTransform.ColumnInfo("AL", "ConvL", DataKind.R4),
new ConvertingTransform.ColumnInfo("AM", "ConvM", DataKind.R4),
new ConvertingTransform.ColumnInfo("AN", "ConvN", DataKind.R4)
}
);
TestEstimatorCore(allTypesPipe, allTypesDataView);
var outputPath = GetOutputPath("Convert", "Types.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
var savedData = TakeFilter.Create(Env, allTypesPipe.Fit(allTypesDataView).Transform(allTypesDataView), 2);
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Convert", "Types.tsv");
Done();
}
void TestConcat()
{
string dataPath = GetDataPath("adult.tiny.with-schema.txt");
var source = new MultiFileSource(dataPath);
var loader = new TextLoader(Env, new TextLoader.Arguments
{
Columns = new[] {
new TextLoader.Column("float1", DataKind.R4, 9),
new TextLoader.Column("float4", DataKind.R4, new[] { new TextLoader.Range(9), new TextLoader.Range(10), new TextLoader.Range(11), new TextLoader.Range(12) }),
new TextLoader.Column("float6", DataKind.R4, new[] { new TextLoader.Range(9), new TextLoader.Range(10), new TextLoader.Range(11), new TextLoader.Range(12, 14) }),
new TextLoader.Column("vfloat", DataKind.R4, new[] { new TextLoader.Range(14, null)
{
AutoEnd = false, VariableEnd = true
} })
},
Separator = "\t",
HasHeader = true
}, new MultiFileSource(dataPath));
var data = loader.Read(source);
ColumnType GetType(Schema schema, string name)
{
Assert.True(schema.TryGetColumnIndex(name, out int cIdx), $"Could not find '{name}'");
return(schema[cIdx].Type);
}
var pipe = new ColumnConcatenatingEstimator(Env, "f1", "float1")
.Append(new ColumnConcatenatingEstimator(Env, "f2", "float1", "float1"))
.Append(new ColumnConcatenatingEstimator(Env, "f3", "float4", "float1"))
.Append(new ColumnConcatenatingEstimator(Env, "f4", "float6", "vfloat", "float1"));
data = TakeFilter.Create(Env, data, 10);
data = pipe.Fit(data).Transform(data);
ColumnType t;
t = GetType(data.Schema, "f1");
Assert.True(t is VectorType vt1 && vt1.ItemType == NumberType.R4 && vt1.Size == 1);
t = GetType(data.Schema, "f2");
Assert.True(t is VectorType vt2 && vt2.ItemType == NumberType.R4 && vt2.Size == 2);
t = GetType(data.Schema, "f3");
Assert.True(t is VectorType vt3 && vt3.ItemType == NumberType.R4 && vt3.Size == 5);
t = GetType(data.Schema, "f4");
Assert.True(t is VectorType vt4 && vt4.ItemType == NumberType.R4 && vt4.Size == 0);
data = ColumnSelectingTransformer.CreateKeep(Env, data, new[] { "f1", "f2", "f3", "f4" });
var subdir = Path.Combine("Transform", "Concat");
var outputPath = GetOutputPath(subdir, "Concat1.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true, Dense = true
});
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, data, fs, keepHidden: false);
}
CheckEquality(subdir, "Concat1.tsv");
Done();
}
public void TakeFilterTakesTheFirstNCharacters()
{
var filter = new TakeFilter(1);
Assert.Equal("F", filter.Apply("Foo"));
}
public void TakeFilterThrowsExceptionIfStringIsLessThanRequestedChars()
{
var filter = new TakeFilter(4);
Assert.Throws <InvalidOperationException>(() => filter.Apply("Foo"));
}
void TestConcat()
{
string dataPath = GetDataPath("adult.test");
var source = new MultiFileSource(dataPath);
var loader = new TextLoader(Env, new TextLoader.Arguments
{
Column = new[] {
new TextLoader.Column("float1", DataKind.R4, 0),
new TextLoader.Column("float4", DataKind.R4, new[] { new TextLoader.Range(0), new TextLoader.Range(2), new TextLoader.Range(4), new TextLoader.Range(10) }),
new TextLoader.Column("vfloat", DataKind.R4, new[] { new TextLoader.Range(0), new TextLoader.Range(2), new TextLoader.Range(4), new TextLoader.Range(10, null)
{
AutoEnd = false, VariableEnd = true
} })
},
Separator = ",",
HasHeader = true
}, new MultiFileSource(dataPath));
var data = loader.Read(source);
ColumnType GetType(ISchema schema, string name)
{
Assert.True(schema.TryGetColumnIndex(name, out int cIdx), $"Could not find '{name}'");
return(schema.GetColumnType(cIdx));
}
var pipe = new ConcatEstimator(Env, "f1", "float1")
.Append(new ConcatEstimator(Env, "f2", "float1", "float1"))
.Append(new ConcatEstimator(Env, "f3", "float4", "float1"))
.Append(new ConcatEstimator(Env, "f4", "vfloat", "float1"));
data = TakeFilter.Create(Env, data, 10);
data = pipe.Fit(data).Transform(data);
ColumnType t;
t = GetType(data.Schema, "f1");
Assert.True(t.IsVector && t.ItemType == NumberType.R4 && t.VectorSize == 1);
t = GetType(data.Schema, "f2");
Assert.True(t.IsVector && t.ItemType == NumberType.R4 && t.VectorSize == 2);
t = GetType(data.Schema, "f3");
Assert.True(t.IsVector && t.ItemType == NumberType.R4 && t.VectorSize == 5);
t = GetType(data.Schema, "f4");
Assert.True(t.IsVector && t.ItemType == NumberType.R4 && t.VectorSize == 0);
data = new ChooseColumnsTransform(Env, data, "f1", "f2", "f3", "f4");
var subdir = Path.Combine("Transform", "Concat");
var outputPath = GetOutputPath(subdir, "Concat1.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true, Dense = true
});
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, data, fs, keepHidden: false);
}
CheckEquality(subdir, "Concat1.tsv");
Done();
}
