void ParseNumberValue() { int start = _startPosition; if (_numberBase != 10) { start += 2; } int stop = InputPosition; if (_typeSuffix != null) { stop -= _typeSuffix.Name.Length; } UString digits = CharSource.Slice(start, stop - start); string error; if ((_value = LesLexer.ParseNumberCore(digits, false, _numberBase, _isFloat, _typeSuffix, out error)) == null) { _value = 0; } else if (_value == CodeSymbols.Sub) { InputPosition = _startPosition + 1; _type = TT.Sub; } if (error != null) { Error(_startPosition, error); } }
static string UnescapeQuotedString(ref UString source, bool isVerbatim, Action <int, string> onError, UString indentation) { Debug.Assert(source.Length >= 1); if (isVerbatim) { bool fail; char stringType = (char)source.PopFront(out fail); StringBuilder sb = new StringBuilder(); int c; for (;;) { c = source.PopFront(out fail); if (fail) { break; } if (c == stringType) { if ((c = source.PopFront(out fail)) != stringType) { break; } } sb.Append((char)c); } return(sb.ToString()); } else { // triple-quoted or normal string: let LES lexer handle it return(LesLexer.UnescapeQuotedString(ref source, onError, indentation)); } }
private void TestLes(string input, string expectOutput, int expectMessages = 0, Severity expectSev = 0) { using (G.PushTLV(Token.ToStringStrategyTLV, TokenExt.ToString)) { MessageHolder errorList; var input2 = StripInitialNewline(input); var lexer = new LesLexer(input2, errorList = new MessageHolder()); var wrapr = new LesIndentTokenGenerator(lexer); var output = new DList <Token>(); for (var t = wrapr.NextToken(); t.HasValue; t = wrapr.NextToken()) { output.Add(t.Value); } var expectTokens = new LesLexer(expectOutput, MessageSink.Console).Buffered().Select(t => t.Type() == TokenType.LBrack ? t.WithType((int)TokenType.Indent) : t.Type() == TokenType.RBrack ? t.WithType((int)TokenType.Dedent) : t).ToList(); AreEqual(expectMessages, errorList.List.Count); if (expectMessages > 0) { AreEqual(expectSev, errorList.List.Max(m => m.Severity)); } ExpectList(output, expectTokens, false); } }
List <Token> Lex(string input, bool skipWS = true) { var lexer = new LesLexer(input, MessageSink.Trace); var lexer2 = new TokensToTree(lexer, skipWS); var list = new List <Token>(); Token?token; while ((token = lexer2.NextToken()) != null) { list.Add(token.Value); } return(list); }
int MeasureIndent(UString indent) { return(LesLexer.MeasureIndent(indent, SpacesPerTab)); }