public TokenStream GenerateTokens() { StateStorage scannerStates = new StateStorage(); TokenConstruction constr = new TokenConstruction(); IScannerState currentState = scannerStates.Base; while (true) { source.MoveNext(); if (source.EndOfStream) { break; } char curr = source.Current; if (curr == '\n') { constr.IncrementLines(); } if (!currentState.Equals(scannerStates.StringLiteral) && !currentState.Equals(scannerStates.EscapeCharacter) && !currentState.Equals(scannerStates.StringLiteralStart)) { curr = char.ToLowerInvariant(curr); } currentState = currentState.Read(constr, curr, scannerStates); } if (!currentState.Equals(scannerStates.Base)) { throw new LexerException("Unexpected end of input", constr.Line); } constr.End(Symbol.EndOfInput); return(constr.CreateStream()); }
private static IScannerState Scanner(IScannerState End, string Moves) { IScannerState curr = End; for (int i = Moves.Length - 1; i > 0; i--) { curr = new SingleState(Moves[i], curr); } return(curr); }
public StateStorage() { CommentLevel level = new CommentLevel(); CommentOpen = new CommentOpen(level); CommentClose = new CommentClose(level); Write = Scanner(Symbol.PrintProcedure, "riteln"); Variable = Scanner(Symbol.Variable, "var"); End = Scanner(Symbol.End, "nd"); Boolean = Scanner(Symbol.BooleanType, "oolean"); Do = Scanner(Symbol.Do, "do"); PROcedure = Scanner(Symbol.Procedure, "cedure"); Function = Scanner(Symbol.Function, "function"); PROgram = Scanner(Symbol.Program, "gram"); Begin = Scanner(Symbol.Begin, "egin"); While = Scanner(Symbol.While, "hile"); Else = Scanner(Symbol.Else, "lse"); Of = Scanner(Symbol.Of, "of"); And = Scanner(Symbol.LogicalAnd, "and"); Or = Scanner(Symbol.LogicalOr, "or"); Not = Scanner(Symbol.LogicalNot, "not"); Integer = Scanner(Symbol.IntegerType, "nteger"); T = new CharacterSplit(Value("hen", Symbol.Then), Value("rue", Symbol.BooleanLiteral)); F = new CharacterSplit(Value("alse", Symbol.BooleanLiteral), Value("unction", Symbol.Function)); IScannerState pro = new Pro(); P = Scanner(pro, "pro"); R = new SingleState('e', new CharacterSplit(Value("turn", Symbol.Return), Tuple.Create <char, IScannerState>('a', new CharacterSplit(Value("l", Symbol.RealType), Value("d", Symbol.ReadProcedure))))); IScannerState realRead = new CharacterSplit(Value("l", Symbol.RealType), Value("d", Symbol.ReadProcedure)); REA = Scanner(realRead, "rea"); S = new CharacterSplit(Value("tring", Symbol.StringType), Value("ize", Symbol.Size)); O = new CharacterSplit(Value("f", Symbol.Of), Value("r", Symbol.LogicalOr)); A = new CharacterSplit(Value("nd", Symbol.LogicalAnd), Value("ssert", Symbol.Assert), Value("rray", Symbol.Array)); }
public ScannerTransition(IEnumerable <CharRange> ranges, IScannerState destination) { this.CharRanges = new ReadOnlyCollection <CharRange>(ranges.ToArray()); this.Destination = destination; }
public ScannerTransition(IEnumerable<CharRange> ranges, IScannerState destination) { this.CharRanges = new ReadOnlyCollection<CharRange>(ranges.ToArray()); this.Destination = destination; }
public SingleState(char NextChar, IScannerState NextState) { nextChar = NextChar; nextState = NextState; }