public SyntaxTree Parse(ITextSnapshot snapshot)
        {
            RobotsTxtDocumentSyntax root = new RobotsTxtDocumentSyntax()
            {
                Snapshot = snapshot
            };

            List <SnapshotToken>  leadingTrivia = new List <SnapshotToken>();
            RobotsTxtRecordSyntax currentRecord = new RobotsTxtRecordSyntax()
            {
                Document = root
            };
            bool lastLineWasBlankLine = false;

            foreach (ITextSnapshotLine line in snapshot.Lines)
            {
                bool isBlankLine = false;

                SnapshotPoint cursor = line.Start;
                snapshot.ReadWhiteSpace(ref cursor); // skip white space

                // skip blank lines
                if (cursor == line.End)
                {
                    if (currentRecord.Lines.Any())
                    {
                        root.Records.Add(currentRecord);
                        currentRecord = new RobotsTxtRecordSyntax {
                            Document = root
                        };
                    }

                    continue;
                }

                char first = cursor.GetChar();

                // comment
                if (first == RobotsTxtSyntaxFacts.Comment)
                {
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
                    leadingTrivia.Add(commentToken);
                }

                // record
                else if (Char.IsLetter(first))
                {
                    SnapshotToken name = new SnapshotToken(snapshot.ReadFieldName(ref cursor), _recordNameType);

                    // handle new record
                    if (lastLineWasBlankLine)
                    {
                        if (currentRecord.Lines.Any())
                        {
                            root.Records.Add(currentRecord);
                            currentRecord = new RobotsTxtRecordSyntax {
                                Document = root
                            };
                        }

                        isBlankLine = true;
                    }

                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken delimiter = new SnapshotToken(snapshot.ReadDelimiter(ref cursor), _delimiterType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken value = new SnapshotToken(snapshot.ReadFieldValue(ref cursor), _recordValueType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);

                    IList <SnapshotToken> trailingTrivia = new List <SnapshotToken>();
                    if (!commentToken.IsMissing)
                    {
                        trailingTrivia.Add(commentToken);
                    }

                    RobotsTxtLineSyntax lineSyntax = new RobotsTxtLineSyntax()
                    {
                        Record         = currentRecord,
                        LeadingTrivia  = leadingTrivia,
                        NameToken      = name,
                        DelimiterToken = delimiter,
                        ValueToken     = value,
                        TrailingTrivia = trailingTrivia,
                    };
                    currentRecord.Lines.Add(lineSyntax);
                    leadingTrivia = new List <SnapshotToken>();
                }

                // error
                else
                {
                    ; // TODO: report error
                }
                lastLineWasBlankLine = isBlankLine;
            }

            if (leadingTrivia.Any())
            {
                if (currentRecord.Lines.Any())
                {
                    foreach (var trivia in leadingTrivia)
                    {
                        currentRecord.TrailingTrivia.Add(trivia);
                    }
                }
                else
                {
                    foreach (var trivia in leadingTrivia)
                    {
                        root.LeadingTrivia.Add(trivia);
                    }
                }
            }

            if (currentRecord.Lines.Any())
            {
                root.Records.Add(currentRecord);
            }

            return(new SyntaxTree(snapshot, root));
        }
        public SyntaxTree Parse(ITextSnapshot snapshot)
        {
            RobotsTxtDocumentSyntax root = new RobotsTxtDocumentSyntax() { Snapshot = snapshot };

            List<SnapshotToken> leadingTrivia = new List<SnapshotToken>();
            RobotsTxtRecordSyntax currentRecord = new RobotsTxtRecordSyntax() { Document = root };
            bool lastLineWasBlankLine = false;

            foreach (ITextSnapshotLine line in snapshot.Lines)
            {
                bool isBlankLine = false;

                SnapshotPoint cursor = line.Start;
                snapshot.ReadWhiteSpace(ref cursor); // skip white space

                // skip blank lines
                if (cursor == line.End)
                {
                    if (currentRecord.Lines.Any())
                    {
                        root.Records.Add(currentRecord);
                        currentRecord = new RobotsTxtRecordSyntax { Document = root };
                    }

                    continue;
                }

                char first = cursor.GetChar();

                // comment
                if (first == RobotsTxtSyntaxFacts.Comment)
                {
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
                    leadingTrivia.Add(commentToken);
                }

                // record
                else if (Char.IsLetter(first))
                {
                    SnapshotToken name = new SnapshotToken(snapshot.ReadFieldName(ref cursor), _recordNameType);

                    // handle new record
                    if (lastLineWasBlankLine)
                    {
                        if (currentRecord.Lines.Any())
                        {
                            root.Records.Add(currentRecord);
                            currentRecord = new RobotsTxtRecordSyntax { Document = root };
                        }

                        isBlankLine = true;
                    }

                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken delimiter = new SnapshotToken(snapshot.ReadDelimiter(ref cursor), _delimiterType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken value = new SnapshotToken(snapshot.ReadFieldValue(ref cursor), _recordValueType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);

                    IList<SnapshotToken> trailingTrivia = new List<SnapshotToken>();
                    if (!commentToken.IsMissing)
                        trailingTrivia.Add(commentToken);

                    RobotsTxtLineSyntax lineSyntax = new RobotsTxtLineSyntax()
                    {
                        Record = currentRecord,
                        LeadingTrivia = leadingTrivia,
                        NameToken = name,
                        DelimiterToken = delimiter,
                        ValueToken = value,
                        TrailingTrivia = trailingTrivia,
                    };
                    currentRecord.Lines.Add(lineSyntax);
                    leadingTrivia = new List<SnapshotToken>();
                }

                // error
                else
                    ; // TODO: report error

                lastLineWasBlankLine = isBlankLine;
            }

            if (leadingTrivia.Any())
            {
                if (currentRecord.Lines.Any())
                    foreach (var trivia in leadingTrivia)
                        currentRecord.TrailingTrivia.Add(trivia);
                else
                    foreach (var trivia in leadingTrivia)
                        root.LeadingTrivia.Add(trivia);
            }

            if (currentRecord.Lines.Any())
                root.Records.Add(currentRecord);

            return new SyntaxTree(snapshot, root);
        }