public void WithAdditionalSplitCharacters_ShouldSetAdditionalSplitCharactersCorrectly() { var builder = new TokenizationOptionsBuilder(); builder.SplitOnCharacters('$', '%', '|'); builder.Build().AdditionalSplitCharacters.Should().BeEquivalentTo(new[] { '$', '%', '|' }); }
public void WithSplittingOnPunctuation_ShouldSetTheSplitOnPunctuationPropertyCorrectly(bool setting) { var builder = new TokenizationOptionsBuilder(); builder.SplitOnPunctuation(setting); builder.Build().SplitOnPunctuation.Should().Be(setting); }
public void WithAccentInsensitivity_ShouldSetTheAccentInsensitivityPropertyCorrectly(bool setting) { var builder = new TokenizationOptionsBuilder(); builder.AccentInsensitive(setting); builder.Build().AccentInsensitive.Should().Be(setting); }
public void WithStemming_ShouldSetTheStemmingPropertyCorrectly(bool setting) { var builder = new TokenizationOptionsBuilder(); builder.WithStemming(setting); builder.Build().Stemming.Should().Be(setting); }
public void WithoutApplyingAnyOptions_ShouldSetDefaultsCorrectly() { var builder = new TokenizationOptionsBuilder(); builder.Build().Should().BeEquivalentTo(new TokenizationOptions(TokenizerKind.PlainText) { AccentInsensitive = true, Stemming = false, AdditionalSplitCharacters = Array.Empty <char>(), CaseInsensitive = true, SplitOnPunctuation = true }); }
public void ConfiguringContentAsXml_ShouldSetTheTokenizationKindToXml() { var builder = new TokenizationOptionsBuilder().XmlContent(); builder.Build().TokenizerKind.Should().Be(TokenizerKind.XmlContent); }