public IEnumerable<_64TassToken> Tokenize(string inputLine)
{
currentState = State.BeforeInstruction;
intructionTokenType = _64TassTokenType.DontKnow;
var scanner = new _64TassScanner(inputLine);
while (scanner.SkipWhitespace())
{
yield return GetNextToken(scanner);
}
}
public IEnumerable <_64TassToken> Tokenize(string inputLine)
{
currentState = State.BeforeInstruction;
intructionTokenType = _64TassTokenType.DontKnow;
var scanner = new _64TassScanner(inputLine);
while (scanner.SkipWhitespace())
{
yield return(GetNextToken(scanner));
}
}
private _64TassToken GetNextToken(_64TassScanner scanner)
{
Contract.Requires(scanner != null);
Contract.Ensures(Contract.Result <_64TassToken>() != null);
int tokenStartPosition;
var lexem = scanner.GetNextLexeme(out tokenStartPosition);
return(new _64TassToken(
this.GetTokenTypeForLexem(lexem),
tokenStartPosition,
lexem));
}
private _64TassToken GetNextToken(_64TassScanner scanner)
{
Contract.Requires(scanner != null);
Contract.Ensures(Contract.Result<_64TassToken>() != null);
int tokenStartPosition;
var lexem = scanner.GetNextLexeme(out tokenStartPosition);
return new _64TassToken(
this.GetTokenTypeForLexem(lexem),
tokenStartPosition,
lexem);
}
