TokeniseLine() public static method

public static TokeniseLine ( string line, string file, int lineNumber ) : IEnumerable
line string
file string
lineNumber int
return IEnumerable
Tokeniser Class Documentation
Exemplo n.º 1
0
        public bool MoveNext()
        {
            if (IsAtEnd)
            {
                return(false);
            }

            ++tokenOffset;

            while (tokenOffset >= readTokens.Count)
            {
                string line = input.ReadLine();

                if (line != null)
                {
                    readTokens.AddRange(Tokeniser.TokeniseLine(line, input.CurrentFile, input.LineNumber));
                }
                else
                {
                    break;
                }
            }

            IsAtEnd = tokenOffset >= readTokens.Count;
            return(!IsAtEnd);
        }
Exemplo n.º 2
0
        public static IEnumerable <Token> Tokenise(IPositionableInputStream input)
        {
label_1:
            string line = input.ReadLine();

            if (line != null)
            {
                using (IEnumerator <Token> enumerator = Tokeniser.TokeniseLine(line, input.CurrentFile, input.LineNumber).GetEnumerator()) {
                    while (enumerator.MoveNext())
                    {
                        Token token = enumerator.Current;
                        yield return(token);
                    }
                    goto label_1;
                }
            }
            else
            {
                yield return(new Token(new FilePosition(input.CurrentFile, input.LineNumber, 0), TokenType.EndOfStream, "END"));
            }
        }