public IEnumerable <Token> Lex(string sourceCode) { ICharacterStream stream = new CharacterStream(sourceCode); var result = new List <Token>(); var lineCounter = 1; _buffer = ""; try { while (stream.HasNext()) { var current = stream.Next(); ReadOnlyCollection <Token> tokens = TryFindMarker(stream); if (!tokens.Any()) { tokens = TryNewLine(stream); if (tokens.Any(token => token.Type == TokenType.NewLine)) { ++lineCounter; } if (!tokens.Any()) { tokens = TryFindSpecial(stream); if (!tokens.Any()) { _buffer += current; } } } result.AddRange(tokens); } } catch (Exception exc) { throw new TranslationException($"Lexer error: {exc.Message}", lineCounter, exc); } result.AddRange(PopBuffer()); return(result); }
public Token NextToken() { if (peekedWords.Count > 0) { return(peekedWords.Dequeue()); } Token res = new Token(); char peekChar = stream.Peek(); while (stream.PeekEquals("//")) { stream.DumpUntil('\n'); stream.Next(); peekChar = stream.Peek(); } while (peekChar == ' ' || peekChar == '\n' || peekChar == '\r' || peekChar == '\t') { if (peekChar == '\n') { TabIndex = 0; StartOfLine = true; } else if (peekChar == '\t' && StartOfLine) { TabIndex++; } stream.Next(); peekChar = stream.Peek(); } res.row = stream.Row; res.column = stream.Column; if (StartOfLine) { res.type = TokenTypes.Punctuation_NewLine; res.value = null; StartOfLine = false; } else if (peekChar == 0) { res.type = TokenTypes.EOF; res.value = null; } else if (IsPunctuation(peekChar)) { res.type = punctuationDictionary[stream.Next()]; res.value = null; } else if (char.IsDigit(peekChar)) { res.type = TokenTypes.Number; res.value = ReadNumber(); } else if (peekChar == '"') { res.type = TokenTypes.String; res.value = ReadString(); } else if (IsOperatorStart(peekChar)) { res.type = operatorDictionary[ReadOperator()]; res.value = null; } else if (IsKeyword(stream.PeekOnly(IsKeywordCharacters))) { res.type = TokenTypes.Keyword; res.value = stream.NextOnly(IsKeywordCharacters); } else if (IsIdentifierStart(peekChar)) { res.type = TokenTypes.Identifier; res.value = stream.NextOnly(IsIdentifier); } else { throw new Exception("Unknown thing.."); } return(res); }