public SyntaxTree Parse(ITextSnapshot snapshot)
        {
            RobotsTxtDocumentSyntax root = new RobotsTxtDocumentSyntax() { Snapshot = snapshot };

            List<SnapshotToken> leadingTrivia = new List<SnapshotToken>();
            RobotsTxtRecordSyntax currentRecord = new RobotsTxtRecordSyntax() { Document = root };
            bool lastLineWasBlankLine = false;

            foreach (ITextSnapshotLine line in snapshot.Lines)
            {
                bool isBlankLine = false;

                SnapshotPoint cursor = line.Start;
                snapshot.ReadWhiteSpace(ref cursor); // skip white space

                // skip blank lines
                if (cursor == line.End)
                {
                    if (currentRecord.Lines.Any())
                    {
                        root.Records.Add(currentRecord);
                        currentRecord = new RobotsTxtRecordSyntax { Document = root };
                    }

                    continue;
                }

                char first = cursor.GetChar();

                // comment
                if (first == RobotsTxtSyntaxFacts.Comment)
                {
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
                    leadingTrivia.Add(commentToken);
                }

                // record
                else if (Char.IsLetter(first))
                {
                    SnapshotToken name = new SnapshotToken(snapshot.ReadFieldName(ref cursor), _recordNameType);

                    // handle new record
                    if (lastLineWasBlankLine)
                    {
                        if (currentRecord.Lines.Any())
                        {
                            root.Records.Add(currentRecord);
                            currentRecord = new RobotsTxtRecordSyntax { Document = root };
                        }

                        isBlankLine = true;
                    }

                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken delimiter = new SnapshotToken(snapshot.ReadDelimiter(ref cursor), _delimiterType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken value = new SnapshotToken(snapshot.ReadFieldValue(ref cursor), _recordValueType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);

                    IList<SnapshotToken> trailingTrivia = new List<SnapshotToken>();
                    if (!commentToken.IsMissing)
                        trailingTrivia.Add(commentToken);

                    RobotsTxtLineSyntax lineSyntax = new RobotsTxtLineSyntax()
                    {
                        Record = currentRecord,
                        LeadingTrivia = leadingTrivia,
                        NameToken = name,
                        DelimiterToken = delimiter,
                        ValueToken = value,
                        TrailingTrivia = trailingTrivia,
                    };
                    currentRecord.Lines.Add(lineSyntax);
                    leadingTrivia = new List<SnapshotToken>();
                }

                // error
                else
                    ; // TODO: report error

                lastLineWasBlankLine = isBlankLine;
            }

            if (leadingTrivia.Any())
            {
                if (currentRecord.Lines.Any())
                    foreach (var trivia in leadingTrivia)
                        currentRecord.TrailingTrivia.Add(trivia);
                else
                    foreach (var trivia in leadingTrivia)
                        root.LeadingTrivia.Add(trivia);
            }

            if (currentRecord.Lines.Any())
                root.Records.Add(currentRecord);

            return new SyntaxTree(snapshot, root);
        }
        public SyntaxTree Parse(ITextSnapshot snapshot)
        {
            RobotsTxtDocumentSyntax root = new RobotsTxtDocumentSyntax()
            {
                Snapshot = snapshot
            };

            List <SnapshotToken>  leadingTrivia = new List <SnapshotToken>();
            RobotsTxtRecordSyntax currentRecord = new RobotsTxtRecordSyntax()
            {
                Document = root
            };
            bool lastLineWasBlankLine = false;

            foreach (ITextSnapshotLine line in snapshot.Lines)
            {
                bool isBlankLine = false;

                SnapshotPoint cursor = line.Start;
                snapshot.ReadWhiteSpace(ref cursor); // skip white space

                // skip blank lines
                if (cursor == line.End)
                {
                    if (currentRecord.Lines.Any())
                    {
                        root.Records.Add(currentRecord);
                        currentRecord = new RobotsTxtRecordSyntax {
                            Document = root
                        };
                    }

                    continue;
                }

                char first = cursor.GetChar();

                // comment
                if (first == RobotsTxtSyntaxFacts.Comment)
                {
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
                    leadingTrivia.Add(commentToken);
                }

                // record
                else if (Char.IsLetter(first))
                {
                    SnapshotToken name = new SnapshotToken(snapshot.ReadFieldName(ref cursor), _recordNameType);

                    // handle new record
                    if (lastLineWasBlankLine)
                    {
                        if (currentRecord.Lines.Any())
                        {
                            root.Records.Add(currentRecord);
                            currentRecord = new RobotsTxtRecordSyntax {
                                Document = root
                            };
                        }

                        isBlankLine = true;
                    }

                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken delimiter = new SnapshotToken(snapshot.ReadDelimiter(ref cursor), _delimiterType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken value = new SnapshotToken(snapshot.ReadFieldValue(ref cursor), _recordValueType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);

                    IList <SnapshotToken> trailingTrivia = new List <SnapshotToken>();
                    if (!commentToken.IsMissing)
                    {
                        trailingTrivia.Add(commentToken);
                    }

                    RobotsTxtLineSyntax lineSyntax = new RobotsTxtLineSyntax()
                    {
                        Record         = currentRecord,
                        LeadingTrivia  = leadingTrivia,
                        NameToken      = name,
                        DelimiterToken = delimiter,
                        ValueToken     = value,
                        TrailingTrivia = trailingTrivia,
                    };
                    currentRecord.Lines.Add(lineSyntax);
                    leadingTrivia = new List <SnapshotToken>();
                }

                // error
                else
                {
                    ; // TODO: report error
                }
                lastLineWasBlankLine = isBlankLine;
            }

            if (leadingTrivia.Any())
            {
                if (currentRecord.Lines.Any())
                {
                    foreach (var trivia in leadingTrivia)
                    {
                        currentRecord.TrailingTrivia.Add(trivia);
                    }
                }
                else
                {
                    foreach (var trivia in leadingTrivia)
                    {
                        root.LeadingTrivia.Add(trivia);
                    }
                }
            }

            if (currentRecord.Lines.Any())
            {
                root.Records.Add(currentRecord);
            }

            return(new SyntaxTree(snapshot, root));
        }
        public SyntaxTree Parse(ITextSnapshot snapshot)
        {
            RegistryDocumentSyntax root = new RegistryDocumentSyntax() { Snapshot = snapshot };

            List<SnapshotToken> leadingTrivia = new List<SnapshotToken>();
            RegistrySectionSyntax section = null;

            foreach (ITextSnapshotLine line in snapshot.Lines)
            {
                SnapshotPoint cursor = line.Start;
                snapshot.ReadWhiteSpace(ref cursor); // skip white space

                // read version
                if (line.LineNumber == 0)
                {
                    SnapshotToken versionToken = new SnapshotToken(snapshot.ReadToCommentOrLineEndWhile(ref cursor, _ => true), _versionType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    root.VersionToken = versionToken;

                    if (snapshot.IsAtExact(cursor, RegistrySyntaxFacts.Comment))
                    {
                        SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
                        leadingTrivia.Add(commentToken);
                    }

                    continue;
                }

                // skip blank lines
                if (cursor == line.End)
                    continue;

                char first = cursor.GetChar();

                // comment
                if (first == RegistrySyntaxFacts.Comment)
                {
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
                    leadingTrivia.Add(commentToken);
                }

                // section
                else if (first == RegistrySyntaxFacts.SectionNameOpeningBracket)
                {
                    if (section != null)
                        root.Sections.Add(section);

                    SnapshotToken deleteToken = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.DeleteKey), _operatorType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken openingBracket = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.SectionNameOpeningBracket), _delimiterType);
                    snapshot.ReadWhiteSpace(ref cursor);

                    // read key
                    SeparatedTokenListSyntax path = new SeparatedTokenListSyntax() { Section = section };

                    SnapshotToken name = new SnapshotToken(snapshot.ReadSectionName(ref cursor), _sectionNameType);
                    path.Tokens.Add(name);
                    snapshot.ReadWhiteSpace(ref cursor);

                    while (snapshot.IsAtExact(cursor, RegistrySyntaxFacts.KeySeparator))
                    {
                        SnapshotToken separator = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.KeySeparator), _sectionNameType);
                        path.Separators.Add(separator);
                        snapshot.ReadWhiteSpace(ref cursor);

                        name = new SnapshotToken(snapshot.ReadSectionName(ref cursor), _sectionNameType);
                        path.Tokens.Add(name);
                        snapshot.ReadWhiteSpace(ref cursor);
                    }

                    SnapshotToken closingBracket = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.SectionNameClosingBracket), _delimiterType);
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);

                    IList<SnapshotToken> trailingTrivia = new List<SnapshotToken>();
                    if (!commentToken.IsMissing)
                        trailingTrivia.Add(commentToken);

                    section = new RegistrySectionSyntax()
                    {
                        Document = root,
                        LeadingTrivia = leadingTrivia,
                        DeleteToken = deleteToken,
                        OpeningBracketToken = openingBracket,
                        NameSyntax = path,
                        ClosingBracketToken = closingBracket,
                        TrailingTrivia = trailingTrivia,
                    };
                    leadingTrivia = new List<SnapshotToken>();
                }

                // property
                else if (Char.IsLetter(first) || first == RegistrySyntaxFacts.Quote)
                {
                    // read key name
                    SnapshotToken nameOpeningQuote = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.Quote), _stringType);
                    SnapshotToken name = new SnapshotToken(snapshot.ReadPropertyName(ref cursor), _propertyNameType);
                    SnapshotToken nameClosingQuote = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.Quote), _stringType);

                    // delimiter
                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken nameValueDelimiter = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.PropertyNameValueDelimiter), _delimiterType);
                    snapshot.ReadWhiteSpace(ref cursor);

                    // delete token
                    SnapshotToken deleteToken = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.DeleteKey), _operatorType);
                    snapshot.ReadWhiteSpace(ref cursor);

                    SnapshotToken valueOpeningQuote = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.Quote), _stringType);

                    // read type
                    SnapshotSpan typeSpan = snapshot.ReadType(ref cursor);

                    bool readType = !typeSpan.IsEmpty && RegistrySyntaxFacts.IsKnownDataTypeNameOrShortcut(typeSpan.GetText());

                    SnapshotToken type = readType ? new SnapshotToken(typeSpan, _typeType) : SnapshotToken.CreateMissing(cursor, _typeType);
                    SnapshotToken typeSpecifierOpeningBrace = readType ? new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.TypeSpecifierOpeningBrace), _typeType) : SnapshotToken.CreateMissing(cursor, _typeType);
                    SnapshotToken typeSpecifier = readType ? new SnapshotToken(snapshot.ReadTypeSpecifier(ref cursor), _typeType) : SnapshotToken.CreateMissing(cursor, _typeType);
                    SnapshotToken typeSpecifierClosingBrace = readType ? new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.TypeSpecifierClosingBrace), _typeType) : SnapshotToken.CreateMissing(cursor, _typeType);

                    SnapshotToken typeValueDelimiter = readType ? new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.TypeValueDelimiter), _delimiterType) : SnapshotToken.CreateMissing(cursor, _typeType);

                    if (!readType)
                        cursor = typeSpan.Start;

                    // read value
                    SnapshotToken value = new SnapshotToken(snapshot.ReadPropertyValue(ref cursor), type.IsMissing ? _stringType : _propertyValueType);
                    SnapshotToken valueClosingQuote = new SnapshotToken(snapshot.ReadExact(ref cursor, RegistrySyntaxFacts.Quote), _stringType);

                    snapshot.ReadWhiteSpace(ref cursor);
                    SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);

                    IList<SnapshotToken> trailingTrivia = new List<SnapshotToken>();
                    if (!commentToken.IsMissing)
                        trailingTrivia.Add(commentToken);

                    RegistryPropertySyntax property = new RegistryPropertySyntax()
                    {
                        Section = section,
                        LeadingTrivia = leadingTrivia,
                        NameOpeningQuoteToken = nameOpeningQuote,
                        NameToken = name,
                        NameClosingQuoteToken = nameClosingQuote,
                        NameValueDelimiterToken = nameValueDelimiter,
                        DeleteToken = deleteToken,
                        ValueOpeningQuoteToken = valueOpeningQuote,
                        TypeToken = type,
                        TypeSpecifierOpeningBraceToken = typeSpecifierOpeningBrace,
                        TypeSpecifierToken = typeSpecifier,
                        TypeSpecifierClosingBraceToken = typeSpecifierClosingBrace,
                        TypeValueDelimiterToken = typeValueDelimiter,
                        ValueToken = value,
                        ValueClosingQuoteToken = valueClosingQuote,
                        TrailingTrivia = trailingTrivia,
                    };
                    section.Properties.Add(property);
                    leadingTrivia = new List<SnapshotToken>();
                }

                // error
                else
                    ; // TODO: report error
            }

            if (section != null && leadingTrivia.Any())
                foreach (var trivia in leadingTrivia)
                    section.TrailingTrivia.Add(trivia);

            if (section != null)
                root.Sections.Add(section);

            return new SyntaxTree(snapshot, root);
        }