private LineTokenization TokenizeLine(JSScanner JSScanner, ITextSnapshot snapshot, object previousLineState, int lineNo) { ITextSnapshotLine line = snapshot.GetLineFromLineNumber(lineNo); SnapshotSpan lineSpan = new SnapshotSpan(snapshot, line.Start, line.LengthIncludingLineBreak); var tcp = new SnapshotSpanSourceCodeReader(lineSpan); JSScanner.Initialize( lineSpan.GetText(), previousLineState, new SourceLocation(0, lineNo + 1, 1) ); try { var tokens = JSScanner.ReadTokens(lineSpan.Length).Select(ToTokenKind).ToArray(); return(new LineTokenization(tokens, JSScanner.CurrentState)); } finally { JSScanner.Uninitialize(); } }
public void SnapshotSpanSourceCodeReaderTest() { var text = "hello world\r\nHello again!"; var buffer = new MockTextBuffer(text); var snapshot = new MockTextSnapshot(buffer, text); var reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, text.Length))); Assert.AreEqual(reader.Snapshot, snapshot); Assert.AreEqual(reader.Position, 0); var line = reader.ReadLine(); Assert.AreEqual(line, "hello world"); line = reader.ReadLine(); Assert.AreEqual(line, "Hello again!"); reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, text.Length))); int ch = reader.Peek(); Assert.AreEqual(ch, (int)'h'); char[] readBuf = new char[text.Length]; reader.Read(readBuf, 0, readBuf.Length); Assert.AreEqual(new string(readBuf), text); reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, text.Length))); Assert.AreEqual(reader.ReadToEnd(), text); reader.Reset(); Assert.AreEqual(reader.ReadToEnd(), text); reader.Close(); Assert.AreEqual(reader.Snapshot, null); }
internal void ParseBuffers(BufferParser bufferParser, Severity indentationSeverity, params ITextSnapshot[] snapshots) { IProjectEntry entry = bufferParser._currentProjEntry; IPythonProjectEntry pyProjEntry = entry as IPythonProjectEntry; List<PythonAst> asts = new List<PythonAst>(); foreach (var snapshot in snapshots) { if (snapshot.TextBuffer.Properties.ContainsProperty(PythonReplEvaluator.InputBeforeReset)) { continue; } if (snapshot.IsReplBufferWithCommand()) { continue; } if (pyProjEntry != null && snapshot.TextBuffer.ContentType.IsOfType(PythonCoreConstants.ContentType)) { PythonAst ast; CollectingErrorSink errorSink; List<TaskProviderItem> commentTasks; var reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length))); ParsePythonCode(snapshot, reader, indentationSeverity, out ast, out errorSink, out commentTasks); if (ast != null) { asts.Add(ast); } // update squiggles for the buffer UpdateErrorsAndWarnings(entry, snapshot, errorSink, commentTasks); } else { // other file such as XAML IExternalProjectEntry externalEntry; if ((externalEntry = (entry as IExternalProjectEntry)) != null) { var snapshotContent = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length))); externalEntry.ParseContent(snapshotContent, new SnapshotCookie(snapshotContent.Snapshot)); _analysisQueue.Enqueue(entry, AnalysisPriority.High); } } } if (pyProjEntry != null) { if (asts.Count > 0) { PythonAst finalAst; if (asts.Count == 1) { finalAst = asts[0]; } else { // multiple ASTs, merge them together finalAst = new PythonAst( new SuiteStatement(asts.Select(ast => ast.Body).ToArray()), new int[0], asts[0].LanguageVersion ); } pyProjEntry.UpdateTree(finalAst, new SnapshotCookie(snapshots[0])); // SnapshotCookie is not entirely right, we should merge the snapshots _analysisQueue.Enqueue(entry, AnalysisPriority.High); } else { // indicate that we are done parsing. PythonAst prevTree; IAnalysisCookie prevCookie; pyProjEntry.GetTreeAndCookie(out prevTree, out prevCookie); pyProjEntry.UpdateTree(prevTree, prevCookie); } } }
internal void ParseFile(IProjectEntry entry, string filename, Stream content, Severity indentationSeverity) { IPythonProjectEntry pyEntry; IExternalProjectEntry externalEntry; TextReader reader = null; ITextSnapshot snapshot = GetOpenSnapshot(entry); string zipFileName = GetZipFileName(entry); string pathInZipFile = GetPathInZipFile(entry); IAnalysisCookie cookie; if (snapshot != null) { cookie = new SnapshotCookie(snapshot); reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, 0, snapshot.Length)); } else if (zipFileName != null) { cookie = new ZipFileCookie(zipFileName, pathInZipFile); } else { cookie = new FileCookie(filename); } if ((pyEntry = entry as IPythonProjectEntry) != null) { PythonAst ast; CollectingErrorSink errorSink; List<TaskProviderItem> commentTasks; if (reader != null) { ParsePythonCode(snapshot, reader, indentationSeverity, out ast, out errorSink, out commentTasks); } else { ParsePythonCode(snapshot, content, indentationSeverity, out ast, out errorSink, out commentTasks); } if (ast != null) { pyEntry.UpdateTree(ast, cookie); } else { // notify that we failed to update the existing analysis pyEntry.UpdateTree(null, null); } // update squiggles for the buffer. snapshot may be null if we // are analyzing a file that is not open UpdateErrorsAndWarnings(entry, snapshot, errorSink, commentTasks); // enqueue analysis of the file if (ast != null) { _analysisQueue.Enqueue(pyEntry, AnalysisPriority.Normal); } } else if ((externalEntry = entry as IExternalProjectEntry) != null) { externalEntry.ParseContent(reader ?? new StreamReader(content), cookie); _analysisQueue.Enqueue(entry, AnalysisPriority.Normal); } }
private void ParseBuffers(BufferParser bufferParser, params ITextSnapshot[] snapshots) { IProjectEntry entry = bufferParser._currentProjEntry; IJsProjectEntry jsProjEntry = entry as IJsProjectEntry; List<JsAst> asts = new List<JsAst>(); foreach (var snapshot in snapshots) { if (snapshot.TextBuffer.Properties.ContainsProperty(NodejsReplEvaluator.InputBeforeReset)) { continue; } if (snapshot.IsReplBufferWithCommand()) { continue; } if (jsProjEntry != null && snapshot.TextBuffer.ContentType.IsOfType(NodejsConstants.Nodejs)) { JsAst ast; CollectingErrorSink errorSink; var reader = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length))); ParseNodejsCode(reader, out ast, out errorSink); if (ast != null) { asts.Add(ast); } // update squiggles for the buffer UpdateErrorsAndWarnings(entry, snapshot, errorSink); } else { // other file such as XAML IExternalProjectEntry externalEntry; if ((externalEntry = (entry as IExternalProjectEntry)) != null) { var snapshotContent = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length))); externalEntry.ParseContent(snapshotContent, new SnapshotCookie(snapshotContent.Snapshot)); if (ShouldEnqueue()) { _analysisQueue.Enqueue(entry, AnalysisPriority.High); } } } } if (jsProjEntry != null) { if (asts.Count > 0) { JsAst finalAst; if (asts.Count == 1) { finalAst = asts[0]; } else { // multiple ASTs, merge them together var block = new Block(default(EncodedSpan)); var statements = new List<Statement>(); foreach (var code in asts) { statements.Add(code.Block); } block.Statements = statements.ToArray(); finalAst = asts.Last().CloneWithNewBlock(block); } jsProjEntry.UpdateTree(finalAst, new SnapshotCookie(snapshots[0])); // SnapshotCookie is not entirely right, we should merge the snapshots if (ShouldEnqueue()) { _analysisQueue.Enqueue(entry, AnalysisPriority.High); } } else { // indicate that we are done parsing. JsAst prevTree; IAnalysisCookie prevCookie; jsProjEntry.GetTreeAndCookie(out prevTree, out prevCookie); jsProjEntry.UpdateTree(prevTree, prevCookie); } } }
public void ParseBuffers(BufferParser bufferParser, Severity indentationSeverity, params ITextSnapshot[] snapshots) { IProjectEntry analysis; lock (_openFiles) { if (!_openFiles.TryGetValue(bufferParser, out analysis)) { return; } } IPythonProjectEntry pyProjEntry = analysis as IPythonProjectEntry; List <PythonAst> asts = new List <PythonAst>(); bool hasErrors = false; foreach (var snapshot in snapshots) { var snapshotContent = new SnapshotSpanSourceCodeReader(new SnapshotSpan(snapshot, new Span(0, snapshot.Length))); if (pyProjEntry != null && snapshot.TextBuffer.ContentType.IsOfType(PythonCoreConstants.ContentType)) { if (!snapshot.IsReplBufferWithCommand()) { PythonAst ast; CollectingErrorSink errorSink; ParsePythonCode(snapshotContent, indentationSeverity, out ast, out errorSink); if (ast != null) { asts.Add(ast); if (errorSink.Errors.Count != 0) { hasErrors = true; } // update squiggles for the buffer var buffer = snapshot.TextBuffer; SimpleTagger <ErrorTag> squiggles = _errorProvider.GetErrorTagger(snapshot.TextBuffer); squiggles.RemoveTagSpans(x => true); TaskProvider provider = GetTaskListProviderForProject(bufferParser._currentProjEntry); AddWarnings(snapshot, errorSink, squiggles, provider); AddErrors(snapshot, errorSink, squiggles, provider); UpdateErrorList(errorSink, buffer.GetFilePath(), provider); } } } else { // other file such as XAML IExternalProjectEntry externalEntry; if ((externalEntry = (analysis as IExternalProjectEntry)) != null) { externalEntry.ParseContent(snapshotContent, new SnapshotCookie(snapshotContent.Snapshot)); _analysisQueue.Enqueue(analysis, AnalysisPriority.High); } } } if ((!hasErrors && asts.Count > 0) || asts.Count > 1) { // only update the AST when we're error free, this way we don't remove // a useful analysis with an incomplete and useless analysis. // If we have more than one AST we're in the REPL - we'll update the // AST in that case as errors won't go away. PythonAst finalAst; if (asts.Count == 1) { finalAst = asts[0]; } else { // multiple ASTs, merge them together List <Statement> bodies = new List <Statement>(); foreach (var ast in asts) { bodies.Add(ast.Body); } finalAst = new PythonAst(new SuiteStatement(bodies.ToArray()), new int[0]); } pyProjEntry.UpdateTree(finalAst, new SnapshotCookie(snapshots[0])); // SnapshotCookie is ot entirely right, we should merge the snapshots _analysisQueue.Enqueue(analysis, AnalysisPriority.High); } }