public void TestPartialScanning()
{
var code1 = "/* hello world ";
var code2 = " goodbye */";
CollectingErrorSink errorSink = new CollectingErrorSink(true);
var scanner = new JSScanner(code1, errorSink, new CodeSettings()
{
AllowShebangLine = true
});
var tokens = scanner.ReadTokens(Int32.MaxValue);
VerifyTokens(
tokens,
new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(0, 1, 1), new SourceLocation(15, 1, 16))
);
scanner.Initialize(code2, scanner.CurrentState, new SourceLocation(code1.Length, 2, 1));
tokens = scanner.ReadTokens(Int32.MaxValue);
VerifyTokens(
tokens,
new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(15, 2, 16), new SourceLocation(28, 2, 29))
);
Assert.IsTrue(scanner.CurrentState.Equals(scanner.CurrentState));
Assert.IsFalse(scanner.CurrentState.Equals(new object()));
Assert.AreEqual(scanner.CurrentState.GetHashCode(), 2);
}
private static List <TokenWithSpan> ReadTokens(string code, bool collectWarnings, params ErrorInfo[] errors)
{
CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
var scanner = new JSScanner(code, errorSink, new CodeSettings()
{
AllowShebangLine = true
});
var tokens = scanner.ReadTokens(Int32.MaxValue);
errorSink.CheckErrors(errors);
return(tokens);
}
public void PartialScanning() {
var code1 = "/* hello world ";
var code2 = " goodbye */";
CollectingErrorSink errorSink = new CollectingErrorSink(true);
var scanner = new JSScanner(code1, errorSink, new CodeSettings() { AllowShebangLine = true });
var tokens = scanner.ReadTokens(Int32.MaxValue);
VerifyTokens(
tokens,
new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(0, 1, 1), new SourceLocation(15, 1, 16))
);
scanner.Initialize(code2, scanner.CurrentState, new SourceLocation(code1.Length, 2, 1));
tokens = scanner.ReadTokens(Int32.MaxValue);
VerifyTokens(
tokens,
new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(15, 2, 16), new SourceLocation(28, 2, 29))
);
Assert.IsTrue(scanner.CurrentState.Equals(scanner.CurrentState));
Assert.IsFalse(scanner.CurrentState.Equals(new object()));
Assert.AreEqual(scanner.CurrentState.GetHashCode(), 2);
}
private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo)
{
ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo);
SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak);
var tcp = new SnapshotSpanSourceCodeReader(lineSpan);
JSScanner.Initialize(
lineSpan.GetText(),
previousLineState,
new SourceLocation(0, lineNo + 1, 1)
);
try {
var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray();
return(new LineTokenization(tokens, JSScanner.CurrentState));
} finally {
JSScanner.Uninitialize();
}
}
private static List<TokenWithSpan> ReadTokens(string code, bool collectWarnings, params ErrorInfo[] errors) {
CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings);
var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true });
var tokens = scanner.ReadTokens(Int32.MaxValue);
errorSink.CheckErrors(errors);
return tokens;
}
private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo) {
ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo);
SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak);
var tcp = new SnapshotSpanSourceCodeReader(lineSpan);
JSScanner.Initialize(
lineSpan.GetText(),
previousLineState,
new SourceLocation(0, lineNo + 1, 1)
);
try {
var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray();
return new LineTokenization(tokens, JSScanner.CurrentState);
} finally {
JSScanner.Uninitialize();
}
}