private LineTokenization TokenizeLine(Tokenizer tokenizer, ITextSnapshot snapshot, object previousLineState, int lineNo) {
ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo);
SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak);
var tcp = new SnapshotSpanSourceCodeReader(lineSpan);
tokenizer.Initialize(previousLineState, tcp, new SourceLocation(line.Start.Position, lineNo + 1, 1));
try {
var tokens = tokenizer.ReadTokens(lineSpan.Length).ToArray();
return new LineTokenization(tokens, tokenizer.CurrentState);
} finally {
tokenizer.Uninitialize();
}
}
