public Reconciler(TextNormalizingEstimator.CaseMode textCase, bool keepDiacritics, bool keepPunctuations, bool keepNumbers)
{
_textCase = textCase;
_keepDiacritics = keepDiacritics;
_keepPunctuations = keepPunctuations;
_keepNumbers = keepNumbers;
}
private void TestCaseMode(IDataView dataView, TestClass[] data, TextNormalizingEstimator.CaseMode caseMode)
{
var options = new TextFeaturizingEstimator.Options()
{
CaseMode = caseMode,
OutputTokensColumnName = "OutputTokens"
};
var pipeline = ML.Transforms.Text.FeaturizeText("Features", options, "A");
var model = pipeline.Fit(dataView);
var engine = model.CreatePredictionEngine <TestClass, TestClass>(ML);
var prediction1 = engine.Predict(data[0]);
var prediction2 = engine.Predict(data[1]);
string expected1 = null;
string expected2 = null;
if (caseMode == TextNormalizingEstimator.CaseMode.Upper)
{
expected1 = data[0].A.ToUpper();
expected2 = data[1].A.ToUpper();
}
else if (caseMode == TextNormalizingEstimator.CaseMode.Lower)
{
expected1 = data[0].A.ToLower();
expected2 = data[1].A.ToLower();
}
else if (caseMode == TextNormalizingEstimator.CaseMode.None)
{
expected1 = data[0].A;
expected2 = data[1].A;
}
Assert.Equal(expected1, string.Join(" ", prediction1.OutputTokens));
Assert.Equal(expected2, string.Join(" ", prediction2.OutputTokens));
}
public TextNormalizer(TextNormalizingEstimator.CaseMode caseMode = TextNormalizingEstimator.CaseMode.Lower,
bool keepDiacritics = false,
bool keepPuncuations = false,
bool keepNumbers = false)
{
InitializeTextNormalizer(caseMode, keepDiacritics, keepPuncuations, keepNumbers);
}
private void InitializeTextNormalizer(TextNormalizingEstimator.CaseMode caseMode = TextNormalizingEstimator.CaseMode.Lower,
bool keepDiacritics = false,
bool keepPuncuations = false,
bool keepNumbers = false)
{
_mlContext = new MLContext();
_emptySamplesList = new List <TextData>();
_emptyDataView = _mlContext.Data.LoadFromEnumerable(_emptySamplesList);
// text normalizer
_normTextPipeline = _mlContext.Transforms.Text.NormalizeText("NormalizedText", "Text",
caseMode,
keepDiacritics: keepDiacritics,
keepPunctuations: keepPuncuations,
keepNumbers: keepNumbers);
_normTextTransformer = _normTextPipeline.Fit(_emptyDataView);
_predictionEngine = _mlContext.Model.CreatePredictionEngine <TextData, TransformedTextData>(_normTextTransformer);
}
/// <summary>
/// Normalizes input text by changing case, removing diacritical marks, punctuation marks and/or numbers.
/// </summary>
/// <param name="input">The column to apply to.</param>
/// <param name="caseMode">Casing text using the rules of the invariant culture.</param>
/// <param name="keepDiacritics">Whether to keep diacritical marks or remove them.</param>
/// <param name="keepPunctuations">Whether to keep punctuation marks or remove them.</param>
/// <param name="keepNumbers">Whether to keep numbers or remove them.</param>
public static Scalar <string> NormalizeText(this Scalar <string> input,
TextNormalizingEstimator.CaseMode caseMode = TextNormalizingEstimator.CaseMode.Lower,
bool keepDiacritics = false,
bool keepPunctuations = true,
bool keepNumbers = true) => new OutPipelineColumn(input, caseMode, keepDiacritics, keepPunctuations, keepNumbers);
public OutPipelineColumn(Scalar <string> input, TextNormalizingEstimator.CaseMode textCase, bool keepDiacritics, bool keepPunctuations, bool keepNumbers)
: base(new Reconciler(textCase, keepDiacritics, keepPunctuations, keepNumbers), input)
{
Input = input;
}
