internal TextSpan (Token start, Token end) { this.startLine = start.StartLine; this.startColumn = start.StartColumn; this.endLine = end.StartLine; this.endColumn = end.StartColumn + end.Width; this.startPosition = end.StartPosition + end.Width; this.endPosition = EndPosition; }
public IdentifierToken(Identifier Spelling, int Width, Token.Type Kind, int StartCharacterPosition, int StartLine, int StartColumn, bool FirstOnLine) :base(Kind, StartCharacterPosition, StartLine, StartColumn, FirstOnLine) { this.Spelling = Spelling; this.width = Width; }
public Token(Token.Type Kind, int StartPosition, int StartLine, int StartColumn, bool FirstOnLine) { this.Kind = Kind; this.StartPosition = StartPosition; this.StartLine = StartLine; this.StartColumn = StartColumn; this.firstOnLine = FirstOnLine; }
public Token InsertSemicolonBefore() { Token semicolon = new Token (Token.Type.Semicolon, StartPosition, StartLine, StartColumn, false); semicolon.next = this; return semicolon; }
private bool CheckSyntaxExpected (Token.Type type) { if (current.Kind == type) return true; //default DiagnosticCode code = DiagnosticCode.SyntaxError; switch (type) { case Token.Type.@case: case Token.Type.@default: code = DiagnosticCode.CaseOrDefaultExpected; break; case Token.Type.Identifier: code = DiagnosticCode.IdentifierExpected; break; case Token.Type.LeftBrace: code = DiagnosticCode.LeftBraceExpected; break; case Token.Type.LeftParenthesis: code = DiagnosticCode.LeftParenExpected; break; case Token.Type.Semicolon: code = DiagnosticCode.SemicolonExpected; break; } if (current.Kind == Token.Type.EndOfInput) syntaxIncomplete = true; Error (code, new TextSpan (current,current)); return false; }
private TokenCategory GetCategory(Token.Type type) { switch (type) { case Token.Type.None: case Token.Type.final: case Token.Type.OctalIntegerLiteral: case Token.Type.HexIntegerLiteral: return TokenCategory.None; case Token.Type.EndOfInput: return TokenCategory.EndOfStream; case Token.Type.LeftBrace: case Token.Type.RightBrace: case Token.Type.LeftParenthesis: case Token.Type.RightParenthesis: case Token.Type.LeftBracket: case Token.Type.RightBracket: case Token.Type.Dot: case Token.Type.Semicolon: case Token.Type.Comma: return TokenCategory.Delimiter; case Token.Type.Less: case Token.Type.Greater: case Token.Type.LessEqual: case Token.Type.GreaterEqual: case Token.Type.EqualEqual: case Token.Type.BangEqual: case Token.Type.EqualEqualEqual: case Token.Type.BangEqualEqual: case Token.Type.Plus: case Token.Type.Minus: case Token.Type.Star: case Token.Type.Percent: case Token.Type.PlusPlus: case Token.Type.MinusMinus: case Token.Type.LessLess: case Token.Type.GreaterGreater: case Token.Type.GreaterGreaterGreater: case Token.Type.Ampersand: case Token.Type.Bar: case Token.Type.Circumflex: case Token.Type.Bang: case Token.Type.Tilda: case Token.Type.AmpersandAmpersand: case Token.Type.BarBar: case Token.Type.Question: case Token.Type.Colon: case Token.Type.Equal: case Token.Type.PlusEqual: case Token.Type.MinusEqual: case Token.Type.StarEqual: case Token.Type.PercentEqual: case Token.Type.LessLessEqual: case Token.Type.GreaterGreaterEqual: case Token.Type.GreaterGreaterGreaterEqual: case Token.Type.AmpersandEqual: case Token.Type.BarEqual: case Token.Type.CircumflexEqual: case Token.Type.Divide: case Token.Type.DivideEqual: return TokenCategory.Operator; case Token.Type.@break: case Token.Type.@else: case Token.Type.@new: case Token.Type.var: case Token.Type.@case: case Token.Type.@finally: case Token.Type.@return: case Token.Type.@void: case Token.Type.@catch: case Token.Type.@for: case Token.Type.@switch: case Token.Type.@while: case Token.Type.@continue: case Token.Type.function: case Token.Type.@this: case Token.Type.with: case Token.Type.@default: case Token.Type.@if: case Token.Type.@throw: case Token.Type.delete: case Token.Type.@in: case Token.Type.@try: case Token.Type.@do: case Token.Type.instanceof: case Token.Type.@typeof: case Token.Type.@abstract: case Token.Type.@enum: case Token.Type.@int: case Token.Type.@short: case Token.Type.boolean: case Token.Type.export: case Token.Type.@interface: case Token.Type.@static: case Token.Type.@byte: case Token.Type.extends: case Token.Type.@long: case Token.Type.super: case Token.Type.@char: case Token.Type.native: case Token.Type.synchronized: case Token.Type.@class: case Token.Type.@float: case Token.Type.packate: case Token.Type.throws: case Token.Type.@const: case Token.Type.@goto: case Token.Type.@private: case Token.Type.transient: case Token.Type.debugger: case Token.Type.implements: case Token.Type.@protected: case Token.Type.@volatile: case Token.Type.@double: case Token.Type.import: case Token.Type.@public: case Token.Type.@null: case Token.Type.@true: case Token.Type.@false: return TokenCategory.Keyword; case Token.Type.NumericLiteral: return TokenCategory.NumericLiteral; case Token.Type.StringLiteral: return TokenCategory.StringLiteral; case Token.Type.RegularExpressionLiteral: return TokenCategory.RegularExpressionLiteral; case Token.Type.Identifier: return TokenCategory.Identifier; case Token.Type.Bad: return TokenCategory.Error; } return TokenCategory.None; }
private Token CreateHexIntegerLiteralToken (int first) { StringBuilder builder = new StringBuilder (); builder.Append ((char)first);//0 builder.Append ((char)ReadChar ()); //x or X (ever tested before) double val = 0; while (Advance ()) { int next = PeekChar (); switch (next) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { builder.Append ((char)next); val = val * 16 + next - '0'; ReadChar (); continue; } case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': { builder.Append ((char)next); val = val * 16 + next - 'a' + 10; ReadChar (); continue; } case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': { builder.Append ((char)next); val = val * 16 + next - 'A' + 10; ReadChar (); continue; } } break; } string result = builder.ToString (); current = new HexIntegerLiteralToken (result, val, position, Line, Column, FirstOnLine); return current; }
private void Next () { current = lexer.GetNext (); }
private Token CreateOctalIntegerLiteralToken (int first) { StringBuilder builder = new StringBuilder (); builder.Append ((char)first);//0 double val = 0; while (Advance ()) { int next = PeekChar (); switch (next) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': { builder.Append ((char)next); val = val * 8 + next - '0'; ReadChar (); continue; } } break; } string result = builder.ToString (); current = new OctalIntegerLiteralToken (result, val ,position, Line, Column, FirstOnLine); return current; }
private Token CreateNumericLiteralToken (int first) { int next = PeekChar (); if (first == '0') { if (next == 'x' || next == 'X') return CreateHexIntegerLiteralToken (first); else { switch (next) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': return CreateOctalIntegerLiteralToken (first); }//else continue to make a numerical literal token } } StringBuilder builder = new StringBuilder (); builder.Append ((char) first); int dot = 0; int exp = 0; while (Advance ()) { next = PeekChar (); switch (next) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { builder.Append ((char) next); ReadChar (); continue; } case '.': { ReadChar (); dot++; builder.Append ((char)next); if (dot > 1) { current = CreateBadToken (builder.ToString(), DiagnosticCode.MalformedNumericLiteral); return current; } continue; } case 'e': case 'E': { builder.Append ((char)next); ReadChar (); exp++; if (exp > 1) { current = CreateBadToken (builder.ToString (), DiagnosticCode.MalformedNumericLiteral); return current; } next = PeekChar (); if (next == '+' || next == '-') { builder.Append ((char)next); ReadChar (); } continue; } } break; } string result = builder.ToString (); current = new NumericLiteralToken (result, position, Line, Column, FirstOnLine); return current; }
private Token CreateIdentOrKeyword (int first) { StringBuilder builder = new StringBuilder (); builder.Append ((char) first); while (Advance ()) { int next = PeekChar (); if (Char.IsLetterOrDigit ((char) next) || next == '$' || next == '_') { builder.Append ((char) next); ReadChar (); continue; } break; } string result = builder.ToString (); Token.Type type; bool keyword = keywords.TryGetValue (result, out type); if (keyword) { current = new Token (type, position, Line, Column, FirstOnLine); } else { this.IDTable.InsertIdentifier (result); Identifier id = new Identifier (result, IDTable); current = new IdentifierToken (id, result.Length, Token.Type.Identifier, position, Line, Column, FirstOnLine); } return current; }
private Token CreateToken (Token.Type tokenType, string value) { current = new Token (tokenType, position, Line, Column, FirstOnLine); return current; }
public RegularExpressionLiteralToken ScanRegularExpression (Token Divide) { int startpos = Divide.StartPosition; int startrow = Divide.StartLine; int startcol = Divide.StartColumn; Token current = Divide[this]; StringBuilder regexp = new StringBuilder (); while (current.Kind == Token.Type.Divide) { //get the value of the current token for (int i = 0; i < current.Width; i++) regexp.Append(source [position + i]); current = Divide[this]; } //TODO flags maybe but not always so must peek and not advance //TODO firstonline return new RegularExpressionLiteralToken (regexp.ToString (), "", position - startpos, startpos, startrow, startcol, false); }