public void WithAdditionalSplitCharacters_ShouldSetAdditionalSplitCharactersCorrectly()
{
var builder = new TokenizationOptionsBuilder();
builder.SplitOnCharacters('$', '%', '|');
builder.Build().AdditionalSplitCharacters.Should().BeEquivalentTo(new[] { '$', '%', '|' });
}
public void WithSplittingOnPunctuation_ShouldSetTheSplitOnPunctuationPropertyCorrectly(bool setting)
{
var builder = new TokenizationOptionsBuilder();
builder.SplitOnPunctuation(setting);
builder.Build().SplitOnPunctuation.Should().Be(setting);
}
public void WithAccentInsensitivity_ShouldSetTheAccentInsensitivityPropertyCorrectly(bool setting)
{
var builder = new TokenizationOptionsBuilder();
builder.AccentInsensitive(setting);
builder.Build().AccentInsensitive.Should().Be(setting);
}
public void WithStemming_ShouldSetTheStemmingPropertyCorrectly(bool setting)
{
var builder = new TokenizationOptionsBuilder();
builder.WithStemming(setting);
builder.Build().Stemming.Should().Be(setting);
}
public async Task InitializeAsync()
{
this.index = new FullTextIndexBuilder <string>().Build();
var wikipediaTests = WikipediaDataLoader.Load(typeof(FullTextIndexTests));
var options = new TokenizationOptionsBuilder().XmlContent().WithStemming().Build();
foreach (var(name, text) in wikipediaTests)
{
await this.index.AddAsync(name, text, options);
}
}
public void WithoutApplyingAnyOptions_ShouldSetDefaultsCorrectly()
{
var builder = new TokenizationOptionsBuilder();
builder.Build().Should().BeEquivalentTo(new TokenizationOptions(TokenizerKind.PlainText)
{
AccentInsensitive = true,
Stemming = false,
AdditionalSplitCharacters = Array.Empty <char>(),
CaseInsensitive = true,
SplitOnPunctuation = true
});
}
public void ConfiguringContentAsXml_ShouldSetTheTokenizationKindToXml()
{
var builder = new TokenizationOptionsBuilder().XmlContent();
builder.Build().TokenizerKind.Should().Be(TokenizerKind.XmlContent);
}
