private void ProcessMultiLine(CandidateDefinition matchingToken)
        {
            var multiLineType = _current.Type;
            var multiLineToken = new System.Text.StringBuilder(_current.Value);

            Match continuation;
            do
            {
                continuation = matchingToken.Definition.MultiLineContinuation.Match(_line);
                if (continuation.Value.Length > 0)
                {
                    AdvanceCurrentToken(
                        continuation,
                        matchingToken.Definition.Type
                    );

                    var linesConsumed = ReadNextLine();
                    multiLineToken.Append(_current.Value);
                    for (int i = 0; i < linesConsumed; i++)
                        multiLineToken.AppendLine();
                }
            }
            while (HasMoreTokens && continuation != null && continuation.Value.Length > 0);

            Match terminationMatch = matchingToken.Definition.MultiLineTerminator.Match(_line);
            if (terminationMatch.Value.Length < 0)
                throw new SyntaxException(String.Format("Failed to find terminal for {0}", matchingToken.Definition.Type));

            AdvanceCurrentToken(terminationMatch, matchingToken.Definition.Type);
            multiLineToken.Append(_current.Value);

            _current = new Token(multiLineToken.ToString(), multiLineType);
        }
        private int ReadNextLine()
        {
            int linesConsumed = 0;
            
            while (_line.Length == 0 && _reader.Peek() != -1)
            {
                if (Position.Column > 1)
                    _current = new Token { Type = TokenType.NewLine };

                _line = _reader.ReadLine();
                Position.NewRow();
                linesConsumed++;
            }

            return linesConsumed;
        }
 private void AdvanceCurrentToken(Match match, TokenType type)
 {
     _current = new Token(match.Value, type);
     _line = _line.Remove(match.Captures[0].Index, match.Captures[0].Length);
     Position.Column += match.Value.Length;
 }
        public override void ReadNextToken()
        {
            _current = null;
            ReadNextLine();

            if (_current != Token.Null && _current.Type == TokenType.NewLine)
                return;

            if (!HasMoreTokens)
                return;

            var m = TokenDefinitions
                .Select((td, i) => new CandidateDefinition(i, td.Regex.Match(_line), td))
                .ToList();

            var matchingToken = GetCandidateDefinition(m);
            if (matchingToken == null)
                throw new SyntaxException();

            if (matchingToken.Definition.IsMultiLine)
                ProcessMultiLine(matchingToken);

            if (Current == Token.Null)
                return;

            bool skippable;
            if ((_skippedTokens.TryGetValue(Current.Type, out skippable)) && skippable)
                ReadNextToken();
        }