void match(Token ter, Error err) { if (next.t == ter.t) move(); else throw new ProgramError(err, ter.ToString(), next.line, next.col); }
public Token Scan() { if (pushed.Count != 0) return pushed.Pop(); skipwhite: while (char.IsWhiteSpace(peek)) { peek = readch(); } if (peek == eof) return Token.Eof; tokencol = col; // [A-Za-z_][A-Za-z0-9_]* if (char.IsLetter(peek) || peek == '_') { System.Text.StringBuilder sb = new System.Text.StringBuilder(); do { sb.Append(peek); peek = readch(); } while (char.IsLetterOrDigit(peek) || peek == '_'); string s = sb.ToString(); // reserve keywords and constants. Return their values literally as tokens if (Words.ContainsKey(s)) return Words[s]; var t = new Token(TokenKind.Identifier, s, line, tokencol); Words.Add(s, t); return t; } else if (peek == '\"') { var sb = new StringBuilder(); do { peek = readch(); switch (peek) { case eof: case '\"': break; default: sb.Append(peek); break; } } while (peek != '\"' && peek != eof); peek = ' '; return new StringLiteral(sb.ToString(), line, tokencol); } else if (peek == '\'') { var sb = new StringBuilder(); do { peek = readch(); switch (peek) { case eof: case '\'': break; default: sb.Append(peek); break; } } while (peek != '\'' && peek != eof); peek = ' '; return new StringLiteral(sb.ToString(), line, tokencol); } else if (peek == '.' || char.IsDigit(peek)) { // returns: a dot or a real. real = .d[.d]*|d[.d]* double d = 0; if (peek == '.') { peek = readch(); if (!char.IsDigit(peek)) { return new Token(TokenKind.Dot, line, tokencol); } // Fall through to the fractional part: peek holds the first digit } else { // integral part (lexeme begins with a digit) do { d = d * 10 + double.Parse(peek.ToString()); peek = readch(); } while (char.IsDigit(peek)); } double p = 10; // fractional part while (char.IsDigit(peek) || peek == '.') { if (peek != '.') { d += double.Parse(peek.ToString()) / p; p *= 10; } peek = readch(); } return new Real(d, line, tokencol); } else if (peek == '$') { peek = readch(); double d = 0; while (char.IsDigit(peek) || (peek >= 'a' && peek <= 'f') || (peek >= 'A' && peek <= 'F')) { d = d * 16 + (double)int.Parse(peek.ToString(), System.Globalization.NumberStyles.AllowHexSpecifier); peek = readch(); } return new Real(d, line, tokencol); } else if (peek == '/') { peek = readch(); switch (peek) { case '=': peek = readch(); return new Token(TokenKind.DivideAssignment, line, tokencol); case '/': do peek = readch(); while (peek != '\n' && peek != '\r' && peek != eof); goto skipwhite; case '*': do { peek = readch(); if (peek == '*') { peek = readch(); if (peek == '/') { peek = readch(); goto skipwhite; } } } while (peek != eof); //error(Token.None, "End-of-file found, '*/' expected"); goto skipwhite; default: return new Token(TokenKind.Divide, line, tokencol); } } else { switch (peek) { case '~': peek = readch(); return new Token(TokenKind.BitwiseComplement, line, tokencol); case '(': peek = readch(); return new Token(TokenKind.OpeningParenthesis, line, tokencol); case ')': peek = readch(); return new Token(TokenKind.ClosingParenthesis, line, tokencol); case '{': peek = readch(); return new Token(TokenKind.OpeningCurlyBrace, line, tokencol); case '}': peek = readch(); return new Token(TokenKind.ClosingCurlyBrace, line, tokencol); case '[': peek = readch(); return new Token(TokenKind.OpeningSquareBracket, line, tokencol); case ']': peek = readch(); return new Token(TokenKind.ClosingSquareBracket, line, tokencol); case ';': peek = readch(); return new Token(TokenKind.Semicolon, line, tokencol); case ',': peek = readch(); return new Token(TokenKind.Comma, line, tokencol); case ':': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.Assignment, ":=", line, tokencol); } return new Token(TokenKind.Colon, line, tokencol); case '=': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.Equality, line, tokencol); } return new Token(TokenKind.Assignment, line, tokencol); case '!': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.Inequality, line, tokencol); } return new Token(TokenKind.Not, line, tokencol); case '*': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.MultiplyAssignment, line, tokencol); } return new Token(TokenKind.Multiply, line, tokencol); case '+': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.AdditionAssignment, line, tokencol); } return new Token(TokenKind.Plus, line, tokencol); case '-': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.SubtractionAssignment, line, tokencol); } return new Token(TokenKind.Minus, line, tokencol); case '&': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.AndAssignment, line, tokencol); } else if (peek == '&') { peek = readch(); return new Token(TokenKind.LogicalAnd, line, tokencol); } return new Token(TokenKind.BitwiseAnd, line, tokencol); case '|': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.OrAssignment, line, tokencol); } else if (peek == '|') { peek = readch(); return new Token(TokenKind.LogicalOr, line, tokencol); } return new Token(TokenKind.BitwiseOr, line, tokencol); case '^': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.XorAssignment, line, tokencol); } else if (peek == '^') { peek = readch(); return new Token(TokenKind.LogicalXor, line, tokencol); } return new Token(TokenKind.BitwiseXor, line, tokencol); case '<': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.LessThanOrEqual, line, tokencol); } else if (peek == '<') { peek = readch(); return new Token(TokenKind.ShiftLeft, line, tokencol); } else if (peek == '>') { peek = readch(); return new Token(TokenKind.Inequality, "<>", line, tokencol); } else return new Token(TokenKind.LessThan, line, tokencol); case '>': peek = readch(); if (peek == '=') { peek = readch(); return new Token(TokenKind.GreaterThanOrEqual, line, tokencol); } else if (peek == '>') { peek = readch(); return new Token(TokenKind.ShiftRight, line, tokencol); } else return new Token(TokenKind.GreaterThan, line, tokencol); default: throw new ProgramError(Error.UnexpectedSymbol, line, tokencol); } } }
// useful for lookahead or: >>, interpreted as > >, as in Array<Array<int> > so PutBack(Token.GreaterThan); public void PutBack(Token t) { pushed.Push(t); }
static void reserve(Token t) { Words.Add(t.lexeme, t); }
void match(Token tok) { match(tok, Error.SymbolExpected); }
void move() { next = l.Scan(); t = next.t; }