private static Token[] TokenizeScript(string script, ILanguageSettings loc = null, Flags flags = Flags.None) { Contracts.AssertValue(script); Contracts.AssertValueOrNull(loc); TexlLexer.Flags lexerFlags = flags.HasFlag(Flags.AllowReplaceableExpressions) ? TexlLexer.Flags.AllowReplaceableTokens : TexlLexer.Flags.None; if (loc == null) { return(TexlLexer.LocalizedInstance.LexSource(script, lexerFlags)); } return(TexlLexer.NewInstance(loc).LexSource(script, lexerFlags)); }
public void TestLexNumbersWithLanguageSettings() { Token[] tokens; tokens = TexlLexer.NewInstance(SomeFrenchLikeSettings()).LexSource("123456,78"); Assert.NotNull(tokens); Assert.Equal(2, tokens.Length); Assert.Equal(TokKind.NumLit, tokens[0].Kind); Assert.Equal(123456.78, tokens[0].As <NumLitToken>().Value); tokens = TexlLexer.NewInstance(SomeRomanianLikeSettings()).LexSource("10`12345"); Assert.NotNull(tokens); Assert.Equal(2, tokens.Length); Assert.Equal(TokKind.NumLit, tokens[0].Kind); Assert.Equal(10.12345, tokens[0].As <NumLitToken>().Value); }
public void TestLexListsWithLanguageSettings() { Token[] tokens; tokens = TexlLexer.NewInstance(SomeFrenchLikeSettings()).LexSource("[1,2;2,3;4]"); Assert.NotNull(tokens); Assert.Equal(8, tokens.Length); Assert.Equal(TokKind.BracketOpen, tokens[0].Kind); Assert.Equal(TokKind.NumLit, tokens[1].Kind); Assert.Equal(1.2, tokens[1].As <NumLitToken>().Value); Assert.Equal(TokKind.Comma, tokens[2].Kind); Assert.Equal(TokKind.NumLit, tokens[3].Kind); Assert.Equal(2.3, tokens[3].As <NumLitToken>().Value); Assert.Equal(TokKind.Comma, tokens[4].Kind); Assert.Equal(TokKind.NumLit, tokens[5].Kind); Assert.Equal(4, tokens[5].As <NumLitToken>().Value); Assert.Equal(TokKind.BracketClose, tokens[6].Kind); Assert.Equal(TokKind.Eof, tokens[7].Kind); }
public void TestLexSemicolonListsWithLanguageSettings() { Token[] tokens; tokens = TexlLexer.NewInstance(SomeFrenchLikeSettings()).LexSource("A ;; B ;; C"); Assert.NotNull(tokens); Assert.Equal(10, tokens.Length); Assert.Equal(TokKind.Ident, tokens[0].Kind); Assert.Equal("A", tokens[0].As <IdentToken>().Name.Value); Assert.Equal(TokKind.Whitespace, tokens[1].Kind); Assert.Equal(TokKind.Semicolon, tokens[2].Kind); Assert.Equal(TokKind.Whitespace, tokens[3].Kind); Assert.Equal(TokKind.Ident, tokens[4].Kind); Assert.Equal("B", tokens[4].As <IdentToken>().Name.Value); Assert.Equal(TokKind.Whitespace, tokens[5].Kind); Assert.Equal(TokKind.Semicolon, tokens[6].Kind); Assert.Equal(TokKind.Whitespace, tokens[7].Kind); Assert.Equal(TokKind.Ident, tokens[8].Kind); Assert.Equal("C", tokens[8].As <IdentToken>().Name.Value); Assert.Equal(TokKind.Eof, tokens[9].Kind); }
public void TestUnsupportedDecimalSeparatorCausesFallback() { Token[] tokens; // Simulate an override of the decimal separator to something that AXL does not support. var oldCulture = CultureInfo.CurrentCulture; var newCulture = new CultureInfo(CultureInfo.CurrentCulture.Name); newCulture.NumberFormat.NumberDecimalSeparator = "+"; CultureInfo.CurrentCulture = newCulture; // The lexer should fall back to the invariant separator. TexlLexer lexer = TexlLexer.NewInstance(null); Assert.Equal(lexer.LocalizedPunctuatorDecimalSeparator, TexlLexer.PunctuatorDecimalSeparatorInvariant); tokens = lexer.LexSource("123456.78"); Assert.NotNull(tokens); Assert.Equal(2, tokens.Length); Assert.Equal(TokKind.NumLit, tokens[0].Kind); Assert.Equal(123456.78, tokens[0].As <NumLitToken>().Value); CultureInfo.CurrentCulture = oldCulture; }