private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo)
{
ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo);
SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak);
var tcp = new SnapshotSpanSourceCodeReader(lineSpan);
JSScanner.Initialize(
lineSpan.GetText(),
previousLineState,
new SourceLocation(0, lineNo + 1, 1)
);
try {
var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray();
return(new LineTokenization(tokens, JSScanner.CurrentState));
} finally {
JSScanner.Uninitialize();
}
}
public void SnapshotSpanSourceCodeReaderTest()
{
var text = "hello world\r\nHello again!";
var buffer = new MockTextBuffer(text);
var snapshot = new MockTextSnapshot(buffer, text);
var reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, text.Length)));
Assert.AreEqual(reader.Snapshot, snapshot);
Assert.AreEqual(reader.Position, 0);
var line = reader.ReadLine();
Assert.AreEqual(line, "hello world");
line = reader.ReadLine();
Assert.AreEqual(line, "Hello again!");
reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, text.Length)));
int ch = reader.Peek();
Assert.AreEqual(ch, (int)'h');
char[] readBuf = new char[text.Length];
reader.Read(readBuf, 0, readBuf.Length);
Assert.AreEqual(new string(readBuf), text);
reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, text.Length)));
Assert.AreEqual(reader.ReadToEnd(), text);
reader.Reset();
Assert.AreEqual(reader.ReadToEnd(), text);
reader.Close();
Assert.AreEqual(reader.Snapshot, null);
}
internal void ParseBuffers(BufferParser bufferParser, Severity indentationSeverity, params ITextSnapshot[] snapshots) {
IProjectEntry entry = bufferParser._currentProjEntry;
IPythonProjectEntry pyProjEntry = entry as IPythonProjectEntry;
List<PythonAst> asts = new List<PythonAst>();
foreach (var snapshot in snapshots) {
if (snapshot.TextBuffer.Properties.ContainsProperty(PythonReplEvaluator.InputBeforeReset)) {
continue;
}
if (snapshot.IsReplBufferWithCommand()) {
continue;
}
if (pyProjEntry != null && snapshot.TextBuffer.ContentType.IsOfType(PythonCoreConstants.ContentType)) {
PythonAst ast;
CollectingErrorSink errorSink;
List<TaskProviderItem> commentTasks;
var reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length)));
ParsePythonCode(snapshot, reader, indentationSeverity, out ast, out errorSink, out commentTasks);
if (ast != null) {
asts.Add(ast);
}
// update squiggles for the buffer
UpdateErrorsAndWarnings(entry, snapshot, errorSink, commentTasks);
} else {
// other file such as XAML
IExternalProjectEntry externalEntry;
if ((externalEntry = (entry as IExternalProjectEntry)) != null) {
var snapshotContent = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length)));
externalEntry.ParseContent(snapshotContent, new SnapshotCookie(snapshotContent.Snapshot));
_analysisQueue.Enqueue(entry, AnalysisPriority.High);
}
}
}
if (pyProjEntry != null) {
if (asts.Count > 0) {
PythonAst finalAst;
if (asts.Count == 1) {
finalAst = asts[0];
} else {
// multiple ASTs, merge them together
finalAst = new PythonAst(
new SuiteStatement(asts.Select(ast => ast.Body).ToArray()),
new int[0],
asts[0].LanguageVersion
);
}
pyProjEntry.UpdateTree(finalAst, new SnapshotCookie(snapshots[0])); // SnapshotCookie is not entirely right, we should merge the snapshots
_analysisQueue.Enqueue(entry, AnalysisPriority.High);
} else {
// indicate that we are done parsing.
PythonAst prevTree;
IAnalysisCookie prevCookie;
pyProjEntry.GetTreeAndCookie(out prevTree, out prevCookie);
pyProjEntry.UpdateTree(prevTree, prevCookie);
}
}
}
internal void ParseFile(IProjectEntry entry, string filename, Stream content, Severity indentationSeverity) {
IPythonProjectEntry pyEntry;
IExternalProjectEntry externalEntry;
TextReader reader = null;
ITextSnapshot snapshot = GetOpenSnapshot(entry);
string zipFileName = GetZipFileName(entry);
string pathInZipFile = GetPathInZipFile(entry);
IAnalysisCookie cookie;
if (snapshot != null) {
cookie = new SnapshotCookie(snapshot);
reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, 0, snapshot.Length));
} else if (zipFileName != null) {
cookie = new ZipFileCookie(zipFileName, pathInZipFile);
} else {
cookie = new FileCookie(filename);
}
if ((pyEntry = entry as IPythonProjectEntry) != null) {
PythonAst ast;
CollectingErrorSink errorSink;
List<TaskProviderItem> commentTasks;
if (reader != null) {
ParsePythonCode(snapshot, reader, indentationSeverity, out ast, out errorSink, out commentTasks);
} else {
ParsePythonCode(snapshot, content, indentationSeverity, out ast, out errorSink, out commentTasks);
}
if (ast != null) {
pyEntry.UpdateTree(ast, cookie);
} else {
// notify that we failed to update the existing analysis
pyEntry.UpdateTree(null, null);
}
// update squiggles for the buffer. snapshot may be null if we
// are analyzing a file that is not open
UpdateErrorsAndWarnings(entry, snapshot, errorSink, commentTasks);
// enqueue analysis of the file
if (ast != null) {
_analysisQueue.Enqueue(pyEntry, AnalysisPriority.Normal);
}
} else if ((externalEntry = entry as IExternalProjectEntry) != null) {
externalEntry.ParseContent(reader ?? new StreamReader(content), cookie);
_analysisQueue.Enqueue(entry, AnalysisPriority.Normal);
}
}
private void ParseBuffers(BufferParser bufferParser, params ITextSnapshot[] snapshots) {
IProjectEntry entry = bufferParser._currentProjEntry;
IJsProjectEntry jsProjEntry = entry as IJsProjectEntry;
List<JsAst> asts = new List<JsAst>();
foreach (var snapshot in snapshots) {
if (snapshot.TextBuffer.Properties.ContainsProperty(NodejsReplEvaluator.InputBeforeReset)) {
continue;
}
if (snapshot.IsReplBufferWithCommand()) {
continue;
}
if (jsProjEntry != null && snapshot.TextBuffer.ContentType.IsOfType(NodejsConstants.Nodejs)) {
JsAst ast;
CollectingErrorSink errorSink;
var reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length)));
ParseNodejsCode(reader, out ast, out errorSink);
if (ast != null) {
asts.Add(ast);
}
// update squiggles for the buffer
UpdateErrorsAndWarnings(entry, snapshot, errorSink);
} else {
// other file such as XAML
IExternalProjectEntry externalEntry;
if ((externalEntry = (entry as IExternalProjectEntry)) != null) {
var snapshotContent = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length)));
externalEntry.ParseContent(snapshotContent, new SnapshotCookie(snapshotContent.Snapshot));
if (ShouldEnqueue()) {
_analysisQueue.Enqueue(entry, AnalysisPriority.High);
}
}
}
}
if (jsProjEntry != null) {
if (asts.Count > 0) {
JsAst finalAst;
if (asts.Count == 1) {
finalAst = asts[0];
} else {
// multiple ASTs, merge them together
var block = new Block(default(EncodedSpan));
var statements = new List<Statement>();
foreach (var code in asts) {
statements.Add(code.Block);
}
block.Statements = statements.ToArray();
finalAst = asts.Last().CloneWithNewBlock(block);
}
jsProjEntry.UpdateTree(finalAst, new SnapshotCookie(snapshots[0])); // SnapshotCookie is not entirely right, we should merge the snapshots
if (ShouldEnqueue()) {
_analysisQueue.Enqueue(entry, AnalysisPriority.High);
}
} else {
// indicate that we are done parsing.
JsAst prevTree;
IAnalysisCookie prevCookie;
jsProjEntry.GetTreeAndCookie(out prevTree, out prevCookie);
jsProjEntry.UpdateTree(prevTree, prevCookie);
}
}
}
public void ParseBuffers(BufferParser bufferParser, Severity indentationSeverity, params ITextSnapshot[] snapshots)
{
IProjectEntry analysis;
lock (_openFiles) {
if (!_openFiles.TryGetValue(bufferParser, out analysis))
{
return;
}
}
IPythonProjectEntry pyProjEntry = analysis as IPythonProjectEntry;
List <PythonAst> asts = new List <PythonAst>();
bool hasErrors = false;
foreach (var snapshot in snapshots)
{
var snapshotContent = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length)));
if (pyProjEntry != null && snapshot.TextBuffer.ContentType.IsOfType(PythonCoreConstants.ContentType))
{
if (!snapshot.IsReplBufferWithCommand())
{
PythonAst ast;
CollectingErrorSink errorSink;
ParsePythonCode(snapshotContent, indentationSeverity, out ast, out errorSink);
if (ast != null)
{
asts.Add(ast);
if (errorSink.Errors.Count != 0)
{
hasErrors = true;
}
// update squiggles for the buffer
var buffer = snapshot.TextBuffer;
SimpleTagger <ErrorTag> squiggles = _errorProvider.GetErrorTagger(snapshot.TextBuffer);
squiggles.RemoveTagSpans(x => true);
TaskProvider provider = GetTaskListProviderForProject(bufferParser._currentProjEntry);
AddWarnings(snapshot, errorSink, squiggles, provider);
AddErrors(snapshot, errorSink, squiggles, provider);
UpdateErrorList(errorSink, buffer.GetFilePath(), provider);
}
}
}
else
{
// other file such as XAML
IExternalProjectEntry externalEntry;
if ((externalEntry = (analysis as IExternalProjectEntry)) != null)
{
externalEntry.ParseContent(snapshotContent, new SnapshotCookie(snapshotContent.Snapshot));
_analysisQueue.Enqueue(analysis, AnalysisPriority.High);
}
}
}
if ((!hasErrors && asts.Count > 0) || asts.Count > 1)
{
// only update the AST when we're error free, this way we don't remove
// a useful analysis with an incomplete and useless analysis.
// If we have more than one AST we're in the REPL - we'll update the
// AST in that case as errors won't go away.
PythonAst finalAst;
if (asts.Count == 1)
{
finalAst = asts[0];
}
else
{
// multiple ASTs, merge them together
List <Statement> bodies = new List <Statement>();
foreach (var ast in asts)
{
bodies.Add(ast.Body);
}
finalAst = new PythonAst(new SuiteStatement(bodies.ToArray()), new int[0]);
}
pyProjEntry.UpdateTree(finalAst, new SnapshotCookie(snapshots[0])); // SnapshotCookie is ot entirely right, we should merge the snapshots
_analysisQueue.Enqueue(analysis, AnalysisPriority.High);
}
}
