private LineTokenization TokenizeLine(Tokenizer tokenizer, ITextSnapshot snapshot, object previousLineState, int lineNo) { ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo); SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak); var tcp = new SnapshotSpanSourceCodeReader(lineSpan); tokenizer.Initialize(previousLineState, tcp, new SourceLocation(line.Start.Position, lineNo + 1, 1)); try { var tokens = tokenizer.ReadTokens(lineSpan.Length).ToArray(); return new LineTokenization(tokens, tokenizer.CurrentState); } finally { tokenizer.Uninitialize(); } }