public void WithSplittingOnPunctuation_ShouldSetTheSplitOnPunctuationPropertyCorrectly(bool setting)
{
var builder = new TokenizerBuilder();
builder.SplitOnPunctuation(setting);
builder.Build().Should().BeOfType <Tokenizer>().Subject.Options.SplitOnPunctuation.Should().Be(setting);
}
public static Tokenizer <AsmToken> Create()
{
var tokenizerBuilder = new TokenizerBuilder <AsmToken>()
.Ignore(Span.WhiteSpace)
.Match(Span.EqualTo("MOVE"), AsmToken.Move, true)
.Match(Span.EqualTo("LOAD"), AsmToken.Load, true)
.Match(Span.EqualTo("STORE"), AsmToken.Store, true)
.Match(Span.EqualTo("ADD"), AsmToken.Add, true)
.Match(Span.EqualTo("SUB"), AsmToken.Subtract, true)
.Match(Span.EqualTo("MULT"), AsmToken.Multiply, true)
.Match(Span.EqualTo("BEQ"), AsmToken.BranchEqual, true)
.Match(Span.EqualTo("BLT"), AsmToken.BranchLessThan, true)
.Match(Span.EqualTo("BLE"), AsmToken.BranchLessThanOrEqualTo, true)
.Match(Character.EqualTo(':'), AsmToken.Colon)
.Match(Character.EqualTo('+'), AsmToken.Plus)
.Match(Character.EqualTo('#'), AsmToken.Hash)
.Match(Character.EqualTo(','), AsmToken.Comma)
.Match(Character.EqualTo('\n'), AsmToken.NewLine)
.Match(Span.Regex(@"\d*"), AsmToken.Number)
.Match(Span.Regex(@"R\d*"), AsmToken.Register, true)
.Match(Span.Regex(@"\w[\w\d]*"), AsmToken.Text, true);
return(tokenizerBuilder.Build());
}
public void WithAdditionalSplitCharacters_ShouldSetAdditionalSplitCharactersCorrectly()
{
var builder = new TokenizerBuilder();
builder.SplitOnCharacters('$', '%', '|');
builder.Build().Should().BeOfType <Tokenizer>().Subject.Options.AdditionalSplitCharacters.Should().BeEquivalentTo(new[] { '$', '%', '|' });
}
public void WithAccentInsensitivity_ShouldSetTheAccentInsensitivityPropertyCorrectly(bool setting)
{
var builder = new TokenizerBuilder();
builder.AccentInsensitive(setting);
builder.Build().Should().BeOfType <Tokenizer>().Subject.Options.AccentInsensitive.Should().Be(setting);
}
public void WithStemming_ShouldSetTheStemmingPropertyCorrectly(bool setting)
{
var builder = new TokenizerBuilder();
builder.WithStemming(setting);
builder.Build().Should().BeOfType <Tokenizer>().Subject.Options.Stemming.Should().Be(setting);
}
public void WithoutApplyingAnyOptions_ShouldSetDefaultsCorrectly()
{
var builder = new TokenizerBuilder();
builder.Build().Should().BeOfType <Tokenizer>().Subject
.Options.Should().BeEquivalentTo(expectedDefaultOptions);
}
public void WithSpecifiedTokenizerFactory_ShouldReturnConfiguredType()
{
var builder = new TokenizerBuilder();
builder.WithFactory(o => new FakeTokenizer(o));
builder.Build().Should().BeOfType <FakeTokenizer>().Subject
.Options.Should().BeEquivalentTo(expectedDefaultOptions);
}
public Tokenizer <TokenType> GetTokenizer() => syntaxBuilder.Build();
public void WithoutChangingTokenizerFactory_ShouldBuildTokenizer()
{
var builder = new TokenizerBuilder();
builder.Build().Should().BeOfType <Tokenizer>();
}