public IEnumerable<_64TassToken> Tokenize(string inputLine) { currentState = State.BeforeInstruction; intructionTokenType = _64TassTokenType.DontKnow; var scanner = new _64TassScanner(inputLine); while (scanner.SkipWhitespace()) { yield return GetNextToken(scanner); } }
public IEnumerable <_64TassToken> Tokenize(string inputLine) { currentState = State.BeforeInstruction; intructionTokenType = _64TassTokenType.DontKnow; var scanner = new _64TassScanner(inputLine); while (scanner.SkipWhitespace()) { yield return(GetNextToken(scanner)); } }
private _64TassToken GetNextToken(_64TassScanner scanner) { Contract.Requires(scanner != null); Contract.Ensures(Contract.Result <_64TassToken>() != null); int tokenStartPosition; var lexem = scanner.GetNextLexeme(out tokenStartPosition); return(new _64TassToken( this.GetTokenTypeForLexem(lexem), tokenStartPosition, lexem)); }
private _64TassToken GetNextToken(_64TassScanner scanner) { Contract.Requires(scanner != null); Contract.Ensures(Contract.Result<_64TassToken>() != null); int tokenStartPosition; var lexem = scanner.GetNextLexeme(out tokenStartPosition); return new _64TassToken( this.GetTokenTypeForLexem(lexem), tokenStartPosition, lexem); }