private static Token[] TokenizeScript(string script, ILanguageSettings loc = null, Flags flags = Flags.None)
{
Contracts.AssertValue(script);
Contracts.AssertValueOrNull(loc);
TexlLexer.Flags lexerFlags = flags.HasFlag(Flags.AllowReplaceableExpressions) ? TexlLexer.Flags.AllowReplaceableTokens : TexlLexer.Flags.None;
if (loc == null)
{
return(TexlLexer.LocalizedInstance.LexSource(script, lexerFlags));
}
return(TexlLexer.NewInstance(loc).LexSource(script, lexerFlags));
}
public void TestLexNumbersWithLanguageSettings()
{
Token[] tokens;
tokens = TexlLexer.NewInstance(SomeFrenchLikeSettings()).LexSource("123456,78");
Assert.NotNull(tokens);
Assert.Equal(2, tokens.Length);
Assert.Equal(TokKind.NumLit, tokens[0].Kind);
Assert.Equal(123456.78, tokens[0].As <NumLitToken>().Value);
tokens = TexlLexer.NewInstance(SomeRomanianLikeSettings()).LexSource("10`12345");
Assert.NotNull(tokens);
Assert.Equal(2, tokens.Length);
Assert.Equal(TokKind.NumLit, tokens[0].Kind);
Assert.Equal(10.12345, tokens[0].As <NumLitToken>().Value);
}
public void TestLexListsWithLanguageSettings()
{
Token[] tokens;
tokens = TexlLexer.NewInstance(SomeFrenchLikeSettings()).LexSource("[1,2;2,3;4]");
Assert.NotNull(tokens);
Assert.Equal(8, tokens.Length);
Assert.Equal(TokKind.BracketOpen, tokens[0].Kind);
Assert.Equal(TokKind.NumLit, tokens[1].Kind);
Assert.Equal(1.2, tokens[1].As <NumLitToken>().Value);
Assert.Equal(TokKind.Comma, tokens[2].Kind);
Assert.Equal(TokKind.NumLit, tokens[3].Kind);
Assert.Equal(2.3, tokens[3].As <NumLitToken>().Value);
Assert.Equal(TokKind.Comma, tokens[4].Kind);
Assert.Equal(TokKind.NumLit, tokens[5].Kind);
Assert.Equal(4, tokens[5].As <NumLitToken>().Value);
Assert.Equal(TokKind.BracketClose, tokens[6].Kind);
Assert.Equal(TokKind.Eof, tokens[7].Kind);
}
public void TestLexSemicolonListsWithLanguageSettings()
{
Token[] tokens;
tokens = TexlLexer.NewInstance(SomeFrenchLikeSettings()).LexSource("A ;; B ;; C");
Assert.NotNull(tokens);
Assert.Equal(10, tokens.Length);
Assert.Equal(TokKind.Ident, tokens[0].Kind);
Assert.Equal("A", tokens[0].As <IdentToken>().Name.Value);
Assert.Equal(TokKind.Whitespace, tokens[1].Kind);
Assert.Equal(TokKind.Semicolon, tokens[2].Kind);
Assert.Equal(TokKind.Whitespace, tokens[3].Kind);
Assert.Equal(TokKind.Ident, tokens[4].Kind);
Assert.Equal("B", tokens[4].As <IdentToken>().Name.Value);
Assert.Equal(TokKind.Whitespace, tokens[5].Kind);
Assert.Equal(TokKind.Semicolon, tokens[6].Kind);
Assert.Equal(TokKind.Whitespace, tokens[7].Kind);
Assert.Equal(TokKind.Ident, tokens[8].Kind);
Assert.Equal("C", tokens[8].As <IdentToken>().Name.Value);
Assert.Equal(TokKind.Eof, tokens[9].Kind);
}
public void TestUnsupportedDecimalSeparatorCausesFallback()
{
Token[] tokens;
// Simulate an override of the decimal separator to something that AXL does not support.
var oldCulture = CultureInfo.CurrentCulture;
var newCulture = new CultureInfo(CultureInfo.CurrentCulture.Name);
newCulture.NumberFormat.NumberDecimalSeparator = "+";
CultureInfo.CurrentCulture = newCulture;
// The lexer should fall back to the invariant separator.
TexlLexer lexer = TexlLexer.NewInstance(null);
Assert.Equal(lexer.LocalizedPunctuatorDecimalSeparator, TexlLexer.PunctuatorDecimalSeparatorInvariant);
tokens = lexer.LexSource("123456.78");
Assert.NotNull(tokens);
Assert.Equal(2, tokens.Length);
Assert.Equal(TokKind.NumLit, tokens[0].Kind);
Assert.Equal(123456.78, tokens[0].As <NumLitToken>().Value);
CultureInfo.CurrentCulture = oldCulture;
}
