public SyntaxTree Parse(ITextSnapshot snapshot)
{
RobotsTxtDocumentSyntax root = new RobotsTxtDocumentSyntax()
{
Snapshot = snapshot
};
List <SnapshotToken> leadingTrivia = new List <SnapshotToken>();
RobotsTxtRecordSyntax currentRecord = new RobotsTxtRecordSyntax()
{
Document = root
};
bool lastLineWasBlankLine = false;
foreach (ITextSnapshotLine line in snapshot.Lines)
{
bool isBlankLine = false;
SnapshotPoint cursor = line.Start;
snapshot.ReadWhiteSpace(ref cursor); // skip white space
// skip blank lines
if (cursor == line.End)
{
if (currentRecord.Lines.Any())
{
root.Records.Add(currentRecord);
currentRecord = new RobotsTxtRecordSyntax {
Document = root
};
}
continue;
}
char first = cursor.GetChar();
// comment
if (first == RobotsTxtSyntaxFacts.Comment)
{
SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
leadingTrivia.Add(commentToken);
}
// record
else if (Char.IsLetter(first))
{
SnapshotToken name = new SnapshotToken(snapshot.ReadFieldName(ref cursor), _recordNameType);
// handle new record
if (lastLineWasBlankLine)
{
if (currentRecord.Lines.Any())
{
root.Records.Add(currentRecord);
currentRecord = new RobotsTxtRecordSyntax {
Document = root
};
}
isBlankLine = true;
}
snapshot.ReadWhiteSpace(ref cursor);
SnapshotToken delimiter = new SnapshotToken(snapshot.ReadDelimiter(ref cursor), _delimiterType);
snapshot.ReadWhiteSpace(ref cursor);
SnapshotToken value = new SnapshotToken(snapshot.ReadFieldValue(ref cursor), _recordValueType);
snapshot.ReadWhiteSpace(ref cursor);
SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
IList <SnapshotToken> trailingTrivia = new List <SnapshotToken>();
if (!commentToken.IsMissing)
{
trailingTrivia.Add(commentToken);
}
RobotsTxtLineSyntax lineSyntax = new RobotsTxtLineSyntax()
{
Record = currentRecord,
LeadingTrivia = leadingTrivia,
NameToken = name,
DelimiterToken = delimiter,
ValueToken = value,
TrailingTrivia = trailingTrivia,
};
currentRecord.Lines.Add(lineSyntax);
leadingTrivia = new List <SnapshotToken>();
}
// error
else
{
; // TODO: report error
}
lastLineWasBlankLine = isBlankLine;
}
if (leadingTrivia.Any())
{
if (currentRecord.Lines.Any())
{
foreach (var trivia in leadingTrivia)
{
currentRecord.TrailingTrivia.Add(trivia);
}
}
else
{
foreach (var trivia in leadingTrivia)
{
root.LeadingTrivia.Add(trivia);
}
}
}
if (currentRecord.Lines.Any())
{
root.Records.Add(currentRecord);
}
return(new SyntaxTree(snapshot, root));
}
public SyntaxTree Parse(ITextSnapshot snapshot)
{
RobotsTxtDocumentSyntax root = new RobotsTxtDocumentSyntax() { Snapshot = snapshot };
List<SnapshotToken> leadingTrivia = new List<SnapshotToken>();
RobotsTxtRecordSyntax currentRecord = new RobotsTxtRecordSyntax() { Document = root };
bool lastLineWasBlankLine = false;
foreach (ITextSnapshotLine line in snapshot.Lines)
{
bool isBlankLine = false;
SnapshotPoint cursor = line.Start;
snapshot.ReadWhiteSpace(ref cursor); // skip white space
// skip blank lines
if (cursor == line.End)
{
if (currentRecord.Lines.Any())
{
root.Records.Add(currentRecord);
currentRecord = new RobotsTxtRecordSyntax { Document = root };
}
continue;
}
char first = cursor.GetChar();
// comment
if (first == RobotsTxtSyntaxFacts.Comment)
{
SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
leadingTrivia.Add(commentToken);
}
// record
else if (Char.IsLetter(first))
{
SnapshotToken name = new SnapshotToken(snapshot.ReadFieldName(ref cursor), _recordNameType);
// handle new record
if (lastLineWasBlankLine)
{
if (currentRecord.Lines.Any())
{
root.Records.Add(currentRecord);
currentRecord = new RobotsTxtRecordSyntax { Document = root };
}
isBlankLine = true;
}
snapshot.ReadWhiteSpace(ref cursor);
SnapshotToken delimiter = new SnapshotToken(snapshot.ReadDelimiter(ref cursor), _delimiterType);
snapshot.ReadWhiteSpace(ref cursor);
SnapshotToken value = new SnapshotToken(snapshot.ReadFieldValue(ref cursor), _recordValueType);
snapshot.ReadWhiteSpace(ref cursor);
SnapshotToken commentToken = new SnapshotToken(snapshot.ReadComment(ref cursor), _commentType);
IList<SnapshotToken> trailingTrivia = new List<SnapshotToken>();
if (!commentToken.IsMissing)
trailingTrivia.Add(commentToken);
RobotsTxtLineSyntax lineSyntax = new RobotsTxtLineSyntax()
{
Record = currentRecord,
LeadingTrivia = leadingTrivia,
NameToken = name,
DelimiterToken = delimiter,
ValueToken = value,
TrailingTrivia = trailingTrivia,
};
currentRecord.Lines.Add(lineSyntax);
leadingTrivia = new List<SnapshotToken>();
}
// error
else
; // TODO: report error
lastLineWasBlankLine = isBlankLine;
}
if (leadingTrivia.Any())
{
if (currentRecord.Lines.Any())
foreach (var trivia in leadingTrivia)
currentRecord.TrailingTrivia.Add(trivia);
else
foreach (var trivia in leadingTrivia)
root.LeadingTrivia.Add(trivia);
}
if (currentRecord.Lines.Any())
root.Records.Add(currentRecord);
return new SyntaxTree(snapshot, root);
}