public void TokenizeWithSeparators()
{
string dataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(dataPath).AsDynamic;
var est = new WordTokenizeEstimator(Env, "text", "words", separators: new[] { ' ', '?', '!', '.', ',' });
var outdata = TakeFilter.Create(Env, est.Fit(data).Transform(data), 4);
var savedData = new ChooseColumnsTransform(Env, outdata, "words");
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
var outputPath = GetOutputPath("Text", "tokenizedWithSeparators.tsv");
using (var ch = Env.Start("save"))
{
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "tokenizedWithSeparators.tsv");
Done();
}
public void NgramWorkout()
{
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(sentimentDataPath);
var invalidData = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(sentimentDataPath);
var est = new WordTokenizeEstimator(Env, "text", "text")
.Append(new TermEstimator(Env, "text", "terms"))
.Append(new NgramEstimator(Env, "terms", "ngrams"))
.Append(new NgramHashEstimator(Env, "terms", "ngramshash"));
// The following call fails because of the following issue
// https://github.com/dotnet/machinelearning/issues/969
// TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "ngrams.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(Env, savedData, "text", "terms", "ngrams", "ngramshash");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "ngrams.tsv");
Done();
}
public void TextTokenizationWorkout()
{
string sentimentDataPath = GetDataPath("wikipedia-detox-250-line-data.tsv");
var data = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadText(1)), hasHeader: true)
.Read(sentimentDataPath);
var invalidData = TextLoader.CreateReader(Env, ctx => (
label: ctx.LoadBool(0),
text: ctx.LoadFloat(1)), hasHeader: true)
.Read(sentimentDataPath);
var est = new WordTokenizeEstimator(Env, "text", "words")
.Append(new CharacterTokenizeEstimator(Env, "text", "chars"))
.Append(new KeyToValueEstimator(Env, "chars"));
TestEstimatorCore(est, data.AsDynamic, invalidInput: invalidData.AsDynamic);
var outputPath = GetOutputPath("Text", "tokenized.tsv");
using (var ch = Env.Start("save"))
{
var saver = new TextSaver(Env, new TextSaver.Arguments {
Silent = true
});
IDataView savedData = TakeFilter.Create(Env, est.Fit(data.AsDynamic).Transform(data.AsDynamic), 4);
savedData = new ChooseColumnsTransform(Env, savedData, "text", "words", "chars");
using (var fs = File.Create(outputPath))
DataSaverUtils.SaveDataView(ch, saver, savedData, fs, keepHidden: true);
}
CheckEquality("Text", "tokenized.tsv");
Done();
}
public void TestOldSavingAndLoading()
{
var data = new[] { new TestClass()
{
A = "This is a good sentence.", B = new string[2] {
"Much words", "Wow So Cool"
}
} };
var dataView = ComponentCreation.CreateDataView(Env, data);
var pipe = new WordTokenizeEstimator(Env, new[] {
new WordTokenizeTransform.ColumnInfo("A", "TokenizeA"),
new WordTokenizeTransform.ColumnInfo("B", "TokenizeB"),
});
var result = pipe.Fit(dataView).Transform(dataView);
var resultRoles = new RoleMappedData(result);
using (var ms = new MemoryStream())
{
TrainUtils.SaveModel(Env, Env.Start("saving"), ms, null, resultRoles);
ms.Position = 0;
var loadedView = ModelFileUtils.LoadTransforms(Env, dataView, ms);
}
}
public void WordTokenizeWorkout()
{
var data = new[] { new TestClass()
{
A = "This is a good sentence.", B = new string[2] {
"Much words", "Wow So Cool"
}
} };
var dataView = ComponentCreation.CreateDataView(Env, data);
var invalidData = new[] { new TestWrong()
{
A = 1, B = new float[2] {
2, 3
}
} };
var invalidDataView = ComponentCreation.CreateDataView(Env, invalidData);
var pipe = new WordTokenizeEstimator(Env, new[] {
new WordTokenizeTransform.ColumnInfo("A", "TokenizeA"),
new WordTokenizeTransform.ColumnInfo("B", "TokenizeB"),
});
TestEstimatorCore(pipe, dataView, invalidInput: invalidDataView);
Done();
}
public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
var h = env.Register(RegistrationName);
h.CheckValue(args, nameof(args));
h.CheckValue(input, nameof(input));
h.CheckUserArg(Utils.Size(args.Column) > 0, nameof(args.Column), "Columns must be specified");
// To each input column to the WordHashBagTransform, a tokenize transform is applied,
// followed by applying WordHashVectorizeTransform.
// Since WordHashBagTransform is a many-to-one column transform, for each
// WordHashBagTransform.Column we may need to define multiple tokenize transform columns.
// NgramHashExtractorTransform may need to define an identical number of HashTransform.Columns.
// The intermediate columns are dropped at the end of using a DropColumnsTransform.
IDataView view = input;
var uniqueSourceNames = NgramExtractionUtils.GenerateUniqueSourceNames(h, args.Column, view.Schema);
Contracts.Assert(uniqueSourceNames.Length == args.Column.Length);
var tokenizeColumns = new WordTokenizeTransform.ColumnInfo[args.Column.Length];
var extractorCols = new NgramHashExtractorTransform.Column[args.Column.Length];
var colCount = args.Column.Length;
List <string> tmpColNames = new List <string>();
for (int iinfo = 0; iinfo < colCount; iinfo++)
{
var column = args.Column[iinfo];
int srcCount = column.Source.Length;
var curTmpNames = new string[srcCount];
Contracts.Assert(uniqueSourceNames[iinfo].Length == args.Column[iinfo].Source.Length);
for (int isrc = 0; isrc < srcCount; isrc++)
{
tokenizeColumns[iinfo] = new WordTokenizeTransform.ColumnInfo(args.Column[iinfo].Source[isrc], curTmpNames[isrc] = uniqueSourceNames[iinfo][isrc]);
}
tmpColNames.AddRange(curTmpNames);
extractorCols[iinfo] =
new NgramHashExtractorTransform.Column
{
Name = column.Name,
Source = curTmpNames,
HashBits = column.HashBits,
NgramLength = column.NgramLength,
Seed = column.Seed,
SkipLength = column.SkipLength,
Ordered = column.Ordered,
InvertHash = column.InvertHash,
FriendlyNames = args.Column[iinfo].Source,
AllLengths = column.AllLengths
};
}
view = new WordTokenizeEstimator(env, tokenizeColumns).Fit(view).Transform(view);
var featurizeArgs =
new NgramHashExtractorTransform.Arguments
{
AllLengths = args.AllLengths,
HashBits = args.HashBits,
NgramLength = args.NgramLength,
SkipLength = args.SkipLength,
Ordered = args.Ordered,
Seed = args.Seed,
Column = extractorCols.ToArray(),
InvertHash = args.InvertHash
};
view = NgramHashExtractorTransform.Create(h, featurizeArgs, view);
// Since we added columns with new names, we need to explicitly drop them before we return the IDataTransform.
var dropColsArgs =
new DropColumnsTransform.Arguments()
{
Column = tmpColNames.ToArray()
};
return(new DropColumnsTransform(h, dropColsArgs, view));
}
public static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
{
Contracts.CheckValue(env, nameof(env));
var h = env.Register(RegistrationName);
h.CheckValue(args, nameof(args));
h.CheckValue(input, nameof(input));
h.CheckUserArg(Utils.Size(args.Column) > 0, nameof(args.Column), "Columns must be specified");
// Compose the WordBagTransform from a tokenize transform,
// followed by a NgramExtractionTransform.
// Since WordBagTransform is a many-to-one column transform, for each
// WordBagTransform.Column with multiple sources, we first apply a ConcatTransform.
// REVIEW: In order to not get ngrams that cross between vector slots, we need to
// enable tokenize transforms to insert a special token between slots.
// REVIEW: In order to make it possible to output separate bags for different columns
// using the same dictionary, we need to find a way to make ConcatTransform remember the boundaries.
var tokenizeColumns = new WordTokenizeTransform.ColumnInfo[args.Column.Length];
var extractorArgs =
new NgramExtractorTransform.Arguments()
{
MaxNumTerms = args.MaxNumTerms,
NgramLength = args.NgramLength,
SkipLength = args.SkipLength,
AllLengths = args.AllLengths,
Weighting = args.Weighting,
Column = new NgramExtractorTransform.Column[args.Column.Length]
};
for (int iinfo = 0; iinfo < args.Column.Length; iinfo++)
{
var column = args.Column[iinfo];
h.CheckUserArg(!string.IsNullOrWhiteSpace(column.Name), nameof(column.Name));
h.CheckUserArg(Utils.Size(column.Source) > 0, nameof(column.Source));
h.CheckUserArg(column.Source.All(src => !string.IsNullOrWhiteSpace(src)), nameof(column.Source));
tokenizeColumns[iinfo] = new WordTokenizeTransform.ColumnInfo(column.Source.Length > 1 ? column.Name : column.Source[0], column.Name);
extractorArgs.Column[iinfo] =
new NgramExtractorTransform.Column()
{
Name = column.Name,
Source = column.Name,
MaxNumTerms = column.MaxNumTerms,
NgramLength = column.NgramLength,
SkipLength = column.SkipLength,
Weighting = column.Weighting,
AllLengths = column.AllLengths
};
}
IDataView view = input;
view = NgramExtractionUtils.ApplyConcatOnSources(h, args.Column, view);
view = new WordTokenizeEstimator(env, tokenizeColumns).Fit(view).Transform(view);
return(NgramExtractorTransform.Create(h, extractorArgs, view));
}
