public static Tokenizer <AsmToken> Create()
{
var tokenizerBuilder = new TokenizerBuilder <AsmToken>()
.Ignore(Span.WhiteSpace)
.Match(Span.EqualTo("MOVE"), AsmToken.Move, true)
.Match(Span.EqualTo("LOAD"), AsmToken.Load, true)
.Match(Span.EqualTo("STORE"), AsmToken.Store, true)
.Match(Span.EqualTo("ADD"), AsmToken.Add, true)
.Match(Span.EqualTo("SUB"), AsmToken.Subtract, true)
.Match(Span.EqualTo("MULT"), AsmToken.Multiply, true)
.Match(Span.EqualTo("BEQ"), AsmToken.BranchEqual, true)
.Match(Span.EqualTo("BLT"), AsmToken.BranchLessThan, true)
.Match(Span.EqualTo("BLE"), AsmToken.BranchLessThanOrEqualTo, true)
.Match(Character.EqualTo(':'), AsmToken.Colon)
.Match(Character.EqualTo('+'), AsmToken.Plus)
.Match(Character.EqualTo('#'), AsmToken.Hash)
.Match(Character.EqualTo(','), AsmToken.Comma)
.Match(Character.EqualTo('\n'), AsmToken.NewLine)
.Match(Span.Regex(@"\d*"), AsmToken.Number)
.Match(Span.Regex(@"R\d*"), AsmToken.Register, true)
.Match(Span.Regex(@"\w[\w\d]*"), AsmToken.Text, true);
return(tokenizerBuilder.Build());
}
public void WithSplittingOnPunctuation_ShouldSetTheSplitOnPunctuationPropertyCorrectly(bool setting)
{
var builder = new TokenizerBuilder();
builder.SplitOnPunctuation(setting);
builder.Build().Should().BeOfType <Tokenizer>().Subject.Options.SplitOnPunctuation.Should().Be(setting);
}
public static Tokenizer <FormulaToken> Create()
{
var tokenizer = new TokenizerBuilder <FormulaToken>()
.Ignore(Span.WhiteSpace)
.Match(Character.EqualTo('='), FormulaToken.Equal)
.Match(Span.EqualTo("<>"), FormulaToken.NotEqual)
.Match(Span.EqualTo(">="), FormulaToken.GreaterEqual)
.Match(Span.EqualTo("<="), FormulaToken.LessEqual)
.Match(Character.EqualTo('>'), FormulaToken.Greater)
.Match(Character.EqualTo('<'), FormulaToken.Less)
.Match(Character.EqualTo('-'), FormulaToken.Minus)
.Match(Character.EqualTo('+'), FormulaToken.Plus)
.Match(Character.EqualTo('*'), FormulaToken.Asterisk)
.Match(Character.EqualTo('/'), FormulaToken.Slash)
.Match(Character.EqualTo(','), FormulaToken.Comma)
.Match(Character.EqualTo('('), FormulaToken.LeftParenthesis)
.Match(Character.EqualTo(')'), FormulaToken.RightParenthesis)
.Match(Character.EqualTo(';'), FormulaToken.Semicolon)
.Match(Span.Regex(@"^-?\d+(,\d+)*(\.\d+(e\d+)?)?"), FormulaToken.Number, true)
.Match(Span.Regex(@"\w[\w\d\s\._áéíóúñÁÉÍÓÚÑ]*"), FormulaToken.Identifier, true)
.Build();
return(tokenizer);
}
public static Tokenizer <SimpleToken> Create()
{
var builder = new TokenizerBuilder <SimpleToken>()
.Match(Span.Regex(@"""((?:""""|[^""])*)"""), SimpleToken.Text)
.Match(ExtraParsers.SpanBetween('<', '>'), SimpleToken.Echo)
.Ignore(Span.WhiteSpace)
.BooleanOperators()
.Match(Character.EqualTo(':'), SimpleToken.Colon)
.Match(Character.EqualTo('!'), SimpleToken.Exclamation)
.Match(Character.EqualTo(','), SimpleToken.Comma)
.Match(Character.EqualTo('='), SimpleToken.Equal)
.Match(Character.EqualTo('('), SimpleToken.OpenParen)
.Match(Character.EqualTo(')'), SimpleToken.CloseParen)
.Match(Character.EqualTo('['), SimpleToken.OpenBracket)
.Match(Character.EqualTo(']'), SimpleToken.CloseBracket)
.Match(Character.EqualTo('{'), SimpleToken.OpenBrace)
.Match(Character.EqualTo('}'), SimpleToken.CloseBrace)
.Match(Character.EqualTo(';'), SimpleToken.Semicolon)
.Match(Span.EqualTo("if"), SimpleToken.If, true)
.Match(Span.EqualTo("else"), SimpleToken.Else, true)
.Match(Numerics.Integer, SimpleToken.Integer)
.Match(Numerics.DecimalDouble, SimpleToken.Double)
.Match(Span.Regex(IdentifierRegex), SimpleToken.Identifier)
.Build();
return(builder);
}
public void WithAdditionalSplitCharacters_ShouldSetAdditionalSplitCharactersCorrectly()
{
var builder = new TokenizerBuilder();
builder.SplitOnCharacters('$', '%', '|');
builder.Build().Should().BeOfType <Tokenizer>().Subject.Options.AdditionalSplitCharacters.Should().BeEquivalentTo(new[] { '$', '%', '|' });
}
public void WithStemming_ShouldSetTheStemmingPropertyCorrectly(bool setting)
{
var builder = new TokenizerBuilder();
builder.WithStemming(setting);
builder.Build().Should().BeOfType <Tokenizer>().Subject.Options.Stemming.Should().Be(setting);
}
public void WithAccentInsensitivity_ShouldSetTheAccentInsensitivityPropertyCorrectly(bool setting)
{
var builder = new TokenizerBuilder();
builder.AccentInsensitive(setting);
builder.Build().Should().BeOfType <Tokenizer>().Subject.Options.AccentInsensitive.Should().Be(setting);
}
public void WithoutApplyingAnyOptions_ShouldSetDefaultsCorrectly()
{
var builder = new TokenizerBuilder();
builder.Build().Should().BeOfType <Tokenizer>().Subject
.Options.Should().BeEquivalentTo(expectedDefaultOptions);
}
public void WithSpecifiedTokenizerFactory_ShouldReturnConfiguredType()
{
var builder = new TokenizerBuilder();
builder.WithFactory(o => new FakeTokenizer(o));
builder.Build().Should().BeOfType <FakeTokenizer>().Subject
.Options.Should().BeEquivalentTo(expectedDefaultOptions);
}
public void InvalidDelimitedTokenAtEndIsReported()
{
var tokenizer = new TokenizerBuilder <string>()
.Match(Span.EqualTo("abc"), "abc", requireDelimiters: true)
.Match(Character.LetterOrDigit.AtLeastOnce(), "lod", requireDelimiters: true)
.Build();
var tokens = tokenizer.TryTokenize("abc_");
Assert.False(tokens.HasValue);
}
private static TokenizerBuilder <SimpleToken> BooleanOperators(this TokenizerBuilder <SimpleToken> builder)
{
builder
.Match(Span.EqualTo("=="), SimpleToken.EqualEqual)
.Match(Span.EqualTo("!="), SimpleToken.NotEqual)
.Match(Span.EqualTo(">="), SimpleToken.GreaterOrEqual)
.Match(Span.EqualTo("<="), SimpleToken.LessOrEqual)
.Match(Character.EqualTo('>'), SimpleToken.Greater)
.Match(Character.EqualTo('<'), SimpleToken.Less)
;
return(builder);
}
public void KeywordsRequireDelimiters()
{
var tokenizer = new TokenizerBuilder <bool>()
.Ignore(Span.WhiteSpace)
.Match(Span.EqualTo("is"), true, requireDelimiters: true)
.Match(Character.Letter.AtLeastOnce(), false, requireDelimiters: true)
.Build();
var tokens = tokenizer.TryTokenize("is isnot is notis ins not is");
Assert.True(tokens.HasValue);
Assert.Equal(7, tokens.Value.Count());
Assert.Equal(3, tokens.Value.Count(v => v.Kind));
}
public void PartiallyFailedTokenizationIsReported()
{
var tokenizer = new TokenizerBuilder <string>()
.Ignore(Span.WhiteSpace)
.Match(Span.EqualTo("abc"), "abc")
.Match(Span.EqualTo("def"), "def")
.Build();
var tokens = tokenizer.TryTokenize(" abd");
Assert.False(tokens.HasValue);
var msg = tokens.ToString();
Assert.Equal("Syntax error (line 1, column 2): invalid abc, unexpected `d`, expected `c` at line 1, column 4.", msg);
}
public void ShortTokenizationIsReported()
{
var tokenizer = new TokenizerBuilder <string>()
.Ignore(Span.WhiteSpace)
.Match(Span.EqualTo("abc"), "abc")
.Match(Span.EqualTo("def"), "def")
.Build();
var tokens = tokenizer.TryTokenize(" ab");
Assert.False(tokens.HasValue);
var msg = tokens.ToString();
Assert.Equal("Syntax error (line 1, column 2): incomplete abc, unexpected end of input, expected `c`.", msg);
}
public void SExpressionsCanBeTokenized()
{
var tokenizer = new TokenizerBuilder <SExpressionToken>()
.Ignore(Span.WhiteSpace)
.Match(Character.EqualTo('('), SExpressionToken.LParen)
.Match(Character.EqualTo(')'), SExpressionToken.RParen)
.Match(Numerics.Integer, SExpressionToken.Number, requireDelimiters: true)
.Match(Character.Letter.IgnoreThen(Character.LetterOrDigit.AtLeastOnce()), SExpressionToken.Atom, requireDelimiters: true)
.Ignore(Comment.ShellStyle)
.Build();
var tokens = tokenizer.TryTokenize("abc (123 def) # this is a comment");
Assert.True(tokens.HasValue);
Assert.Equal(5, tokens.Value.Count());
}
public static Tokenizer <SimpleToken> Create()
{
var tokenizer = new TokenizerBuilder <SimpleToken>()
.Ignore(Span.WhiteSpace)
.Match(Character.EqualTo('('), SimpleToken.LeftParenthesis)
.Match(Character.EqualTo(')'), SimpleToken.RightParenthesis)
.Match(Span.EqualTo("AND"), SimpleToken.And)
.Match(Span.EqualTo("OR"), SimpleToken.Or)
.Match(Span.EqualTo("NOT"), SimpleToken.Not)
.Match(Span.Regex(@"\w*"), SimpleToken.Identifier, true)
.Build();
return(tokenizer);
}
internal SyntaxBuilder(ISettings settings)
{
Builder =
new TokenizerBuilder <TokenType>()
.Ignore(Span.WhiteSpace)
.Match(Character.In(settings.Sets), TokenType.Set)
.Match(Character.In(settings.Unions), TokenType.Union)
.Match(Character.In(settings.Intersections), TokenType.Intersection)
.Match(Character.In(settings.Differences), TokenType.Difference)
.Match(Character.In(settings.SymmetricDifferences), TokenType.SymmetricDifference)
.Match(Character.In(settings.PrefixNegations), TokenType.PrefixNegation)
.Match(Character.In(settings.PostfixNegations), TokenType.PostfixNegation)
.Match(Character.In(settings.LParens), TokenType.LParen)
.Match(Character.In(settings.RParens), TokenType.RParen)
.Match(Character.In(settings.UniverseSets), TokenType.UniverseSet)
.Match(Character.In(settings.EmptySets), TokenType.EmptySet);
}
public SyntaxWithVariables(ISettings settings)
{
var defaultSettings = new DefaultSettings();
syntaxBuilder = new SyntaxBuilder(settings).Builder;
}
public void WithoutChangingTokenizerFactory_ShouldBuildTokenizer()
{
var builder = new TokenizerBuilder();
builder.Build().Should().BeOfType <Tokenizer>();
}
public Syntax(ISettings settings)
{
syntaxBuilder = new SyntaxBuilder(settings).Builder;
}
