public void UpdateModule(IPythonProjectEntry entry, string code) { CollectingErrorSink errors = null; if (code != null) { errors = new CollectingErrorSink(); var parser = Parser.CreateParser( new StringReader(code), _analyzer.LanguageVersion, new ParserOptions { BindReferences = true, ErrorSink = errors } ); using (var p = entry.BeginParse()) { p.Tree = parser.ParseFile(); p.Complete(); } if (errors.Errors.Any() || errors.Warnings.Any()) { if (AssertOnParseErrors) { var errorMsg = MakeMessage(errors); Trace.TraceError(errorMsg); Assert.Fail("Errors parsing " + entry.FilePath, errorMsg); } } } entry.PreAnalyze(); }
/// <summary> /// Attempts to parse the given text. Returns true if the text is a valid expression. Returns false if the text is not /// a valid expression and assigns the error messages produced to errorMsg. /// </summary> public virtual bool TryParseText(string text, out string errorMsg) { CollectingErrorSink errorSink = new CollectingErrorSink(); Parser parser = Parser.CreateParser(new StringReader(text), _thread.Process.LanguageVersion, new ParserOptions() { ErrorSink = errorSink }); var ast = parser.ParseSingleStatement(); if (errorSink.Errors.Count > 0) { StringBuilder msg = new StringBuilder(); foreach (var error in errorSink.Errors) { msg.Append(error.Message); msg.Append(Environment.NewLine); } errorMsg = msg.ToString(); return(false); } errorMsg = null; return(true); }
public void TestPartialScanning() { var code1 = "/* hello world "; var code2 = " goodbye */"; CollectingErrorSink errorSink = new CollectingErrorSink(true); var scanner = new JSScanner(code1, errorSink, new CodeSettings() { AllowShebangLine = true }); var tokens = scanner.ReadTokens(Int32.MaxValue); VerifyTokens( tokens, new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(0, 1, 1), new SourceLocation(15, 1, 16)) ); scanner.Initialize(code2, scanner.CurrentState, new SourceLocation(code1.Length, 2, 1)); tokens = scanner.ReadTokens(Int32.MaxValue); VerifyTokens( tokens, new TokenInfo(JSToken.MultipleLineComment, new SourceLocation(15, 2, 16), new SourceLocation(28, 2, 29)) ); Assert.IsTrue(scanner.CurrentState.Equals(scanner.CurrentState)); Assert.IsFalse(scanner.CurrentState.Equals(new object())); Assert.AreEqual(scanner.CurrentState.GetHashCode(), 2); }
public void UpdateModule(IPythonProjectEntry entry, string code) { CollectingErrorSink errors = null; if (code != null) { PythonAst ast; errors = new CollectingErrorSink(); using (var p = Parser.CreateParser( new StringReader(code), _analyzer.LanguageVersion, new ParserOptions { BindReferences = true, ErrorSink = errors } )) { ast = p.ParseFile(); entry.UpdateTree(ast, null); } if (errors.Errors.Any() || errors.Warnings.Any()) { if (AssertOnParseErrors) { var errorMsg = MakeMessage(errors); Trace.TraceError(errorMsg); Assert.Fail("Errors parsing " + entry.FilePath, errorMsg); } } } entry.Analyze(CancellationToken.None, true); }
public IPythonType GetTypeFromString(string typeString) { // Type alone is not a valid syntax, so we need to simulate the annotation. typeString = $"x: {typeString}"; using (var sr = new StringReader(typeString)) { var sink = new CollectingErrorSink(); var parser = Parser.CreateParser(sr, Module.Interpreter.LanguageVersion, new ParserOptions { ErrorSink = sink }); var ast = parser.ParseFile(); var exprStatement = (ast?.Body as SuiteStatement)?.Statements?.FirstOrDefault() as ExpressionStatement; if (!(Statement.GetExpression(exprStatement) is ExpressionWithAnnotation annExpr) || sink.Errors.Count > 0) { return(null); } var ann = new TypeAnnotation(Ast.LanguageVersion, annExpr.Annotation); var value = ann.GetValue(new TypeAnnotationConverter(this)); var t = value.GetPythonType(); if (!t.IsUnknown()) { return(t); } } return(null); }
public static void Prepare(IPythonProjectEntry entry, TextReader sourceUnit, PythonLanguageVersion version = PythonLanguageVersion.V27) { CollectingErrorSink errorSink = new CollectingErrorSink(); using (var parser = Parser.CreateParser(sourceUnit, errorSink, version)) { entry.UpdateTree(parser.ParseFile(), null); } }
public void Prepare() { _errorSink = new CollectingErrorSink(); // TODO: //using (var parser = Utils.CreateParser(_sourceUnit, _errorSink)) { // Prepare(parser.ParseFile(true)); //} }
private void Parse(CancellationToken cancellationToken) { CollectingErrorSink sink = null; int version; Parser parser; //Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name}"); lock (AnalysisLock) { version = _buffer.Version; var options = new ParserOptions { StubFile = FilePath != null && Path.GetExtension(FilePath).Equals(".pyi", FileSystem.StringComparison) }; if (ModuleType == ModuleType.User) { sink = new CollectingErrorSink(); options.ErrorSink = sink; } parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, options); } var ast = parser.ParseFile(); //Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}"); lock (AnalysisLock) { cancellationToken.ThrowIfCancellationRequested(); if (version != _buffer.Version) { throw new OperationCanceledException(); } _ast = ast; _parseErrors = sink?.Diagnostics ?? Array.Empty <DiagnosticsEntry>(); // Do not report issues with libraries or stubs if (sink != null) { _diagnosticsService?.Replace(Uri, _parseErrors, DiagnosticSource.Parser); } ContentState = State.Parsed; } NewAst?.Invoke(this, EventArgs.Empty); if (ContentState < State.Analyzing) { ContentState = State.Analyzing; var analyzer = Services.GetService <IPythonAnalyzer>(); analyzer.EnqueueDocumentForAnalysis(this, ast, version, _disposeToken.CancellationToken); } lock (AnalysisLock) { _parsingTask = null; } }
private void Parse(CancellationToken cancellationToken) { CollectingErrorSink sink = null; int version; Parser parser; // Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name} ({ModuleType})"); lock (_syncObj) { version = _buffer.Version; var options = new ParserOptions { StubFile = FilePath != null && Path.GetExtension(FilePath).Equals(".pyi", FileSystem.StringComparison) }; if (ModuleType == ModuleType.User) { sink = new CollectingErrorSink(); options.ErrorSink = sink; } parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, options); } var ast = parser.ParseFile(Uri); // Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name} ({ModuleType})"); lock (_syncObj) { cancellationToken.ThrowIfCancellationRequested(); if (version != _buffer.Version) { throw new OperationCanceledException(); } // Stored nodes are no longer valid. _astMap.Clear(); _astMap[this] = ast; _parseErrors = sink?.Diagnostics ?? Array.Empty <DiagnosticsEntry>(); // Do not report issues with libraries or stubs if (sink != null) { _diagnosticsService?.Replace(Uri, _parseErrors, DiagnosticSource.Parser); } ContentState = State.Parsed; Analysis = new EmptyAnalysis(Services, this); } NewAst?.Invoke(this, EventArgs.Empty); Analyze(ast, version); lock (_syncObj) { _parsingTask = null; } }
private static List <TokenWithSpan> ReadTokens(string code, bool collectWarnings, params ErrorInfo[] errors) { CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings); var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true }); var tokens = scanner.ReadTokens(Int32.MaxValue); errorSink.CheckErrors(errors); return(tokens); }
private static void AddWarnings(ITextSnapshot snapshot, CollectingErrorSink errorSink, SimpleTagger <ErrorTag> squiggles, TaskProvider provider) { foreach (ErrorResult warning in errorSink.Warnings) { var span = warning.Span; var tspan = CreateSpan(snapshot, span); squiggles.CreateTagSpan(tspan, new ErrorTag(PredefinedErrorTypeNames.Warning, warning.Message)); if (provider != null) { provider.AddWarning(warning); } } }
private static void AddErrors(ITextSnapshot snapshot, CollectingErrorSink errorSink, SimpleTagger <ErrorTag> squiggles, TaskProvider provider) { foreach (ErrorResult error in errorSink.Errors) { var span = error.Span; var tspan = CreateSpan(snapshot, span); squiggles.CreateTagSpan(tspan, new ErrorTag(PredefinedErrorTypeNames.SyntaxError, error.Message)); if (provider != null) { provider.AddError(error); } } }
public void Parse(TextContentProvider /*!*/ content) { var errorSink = new CollectingErrorSink(); SourceUnitTree ast = MakeParseTree(content, errorSink); ISnapshotTextContentProvider snapshotContent = content as ISnapshotTextContentProvider; if (snapshotContent != null) { // queue analysis of the parsed tree at High Pri so the active buffer is quickly re-analyzed var snapshot = snapshotContent.Snapshot; var analysis = AnalysisItem.GetAnalysis(snapshot.TextBuffer); // only update the AST when we're error free, this way we don't remove // a useful analysis with an incomplete and useless analysis. if (errorSink.Errors.Count == 0) { analysis.UpdateTree(ast, new SnapshotCookie(snapshot)); _analysisQueue.Enqueue(analysis, AnalysisPriority.High); } SimpleTagger <ErrorTag> squiggles = _squiggleProvider.GetErrorTagger(snapshot.TextBuffer); squiggles.RemoveTagSpans(x => true); // update squiggles for the live buffer foreach (ErrorResult warning in errorSink.Warnings) { var span = warning.Span; var tspan = CreateSpan(snapshot, span); squiggles.CreateTagSpan(tspan, new ErrorTag("Warning", warning.Message)); } foreach (ErrorResult error in errorSink.Errors) { var span = error.Span; var tspan = CreateSpan(snapshot, span); squiggles.CreateTagSpan(tspan, new ErrorTag("Error", error.Message)); } } else { FileTextContentProvider fileContent = content as FileTextContentProvider; AnalysisItem analysis; if (fileContent != null && _projectFiles.TryGetValue(fileContent.Path, out analysis)) { analysis.UpdateTree(ast, new FileCookie(fileContent.Path)); _analysisQueue.Enqueue(analysis, AnalysisPriority.Normal); } } }
private void Parse(CancellationToken cancellationToken) { var sink = new CollectingErrorSink(); int version; Parser parser; Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name}"); lock (AnalysisLock) { version = _buffer.Version; parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, new ParserOptions { StubFile = FilePath != null && Path.GetExtension(FilePath).Equals(".pyi", FileSystem.StringComparison), ErrorSink = sink }); } var ast = parser.ParseFile(); Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}"); lock (AnalysisLock) { cancellationToken.ThrowIfCancellationRequested(); if (version != _buffer.Version) { throw new OperationCanceledException(); } _ast = ast; _parseErrors = sink.Diagnostics; _parsingTask = null; } NewAst?.Invoke(this, EventArgs.Empty); if ((_options & ModuleLoadOptions.Analyze) == ModuleLoadOptions.Analyze) { Log?.Log(TraceEventType.Verbose, $"Analysis queued: {Name}"); _linkedAnalysisCts?.Dispose(); _linkedAnalysisCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); var analyzer = Services.GetService <IPythonAnalyzer>(); if (ModuleType == ModuleType.User || ModuleType == ModuleType.Library) { analyzer.AnalyzeDocumentDependencyChainAsync(this, _linkedAnalysisCts.Token).DoNotWait(); } else { analyzer.AnalyzeDocumentAsync(this, _linkedAnalysisCts.Token).DoNotWait(); } } }
private void ParsePythonCode(TextReader content, Severity indentationSeverity, out PythonAst ast, out CollectingErrorSink errorSink) { ast = null; errorSink = new CollectingErrorSink(); using (var parser = MakeParser(content, errorSink, indentationSeverity)) { if (parser != null) { try { ast = parser.ParseFile(); } catch (Exception e) { Debug.Assert(false, String.Format("Failure in Python parser: {0}", e.ToString())); } } } }
private static List <TokenWithSpan> ScanTokens(string code, bool collectWarnings, params ErrorInfo[] errors) { CollectingErrorSink errorSink = new CollectingErrorSink(collectWarnings); var scanner = new JSScanner(code, errorSink, new CodeSettings() { AllowShebangLine = true }); List <TokenWithSpan> tokens = new List <TokenWithSpan>(); for (TokenWithSpan curToken = scanner.ScanNextTokenWithSpan(true); curToken.Token != JSToken.EndOfFile; curToken = scanner.ScanNextTokenWithSpan(true)) { tokens.Add(curToken); } errorSink.CheckErrors(errors); return(tokens); }
private void UpdateErrorList(CollectingErrorSink errorSink, string filepath, TaskProvider provider) { if (_project != null && provider != null) { if (errorSink.Errors.Count > 0) { _project.ErrorFiles.Add(filepath); } else { _project.ErrorFiles.Remove(filepath); } } if (provider != null) { ((IVsTaskList)_errorList).RefreshTasks(provider.Cookie); } }
private void ParsePythonCode(TextContentProvider content, out PythonAst ast, out CollectingErrorSink errorSink) { ast = null; errorSink = new CollectingErrorSink(); // parse the tree var source = _engine.CreateScriptSource(content, "", SourceCodeKind.File); var compOptions = (PythonCompilerOptions)HostingHelpers.GetLanguageContext(_engine).GetCompilerOptions(); var context = new CompilerContext(HostingHelpers.GetSourceUnit(source), compOptions, errorSink); //compOptions.Verbatim = true; using (var parser = MakeParser(context)) { if (parser != null) { try { ast = parser.ParseFile(false); } catch (Exception e) { Debug.Assert(false, String.Format("Failure in IronPython parser: {0}", e.ToString())); } } } }
private static TypeAnnotation Parse(string expr, PythonLanguageVersion version = PythonLanguageVersion.V36) { var errors = new CollectingErrorSink(); var ops = new ParserOptions { ErrorSink = errors }; var p = Parser.CreateParser(new StringReader(expr), version, ops); var ast = p.ParseTopExpression(); if (errors.Errors.Any()) { foreach (var e in errors.Errors) { Console.WriteLine(e); } Assert.Fail(string.Join("\n", errors.Errors.Select(e => e.ToString()))); return(null); } var node = Statement.GetExpression(ast.Body); return(new TypeAnnotation(version, node)); }
private static string MakeMessage(CollectingErrorSink errors) { var sb = new StringBuilder(); if (errors.Errors.Any()) { sb.AppendLine("Errors:"); foreach (var e in errors.Errors) { sb.AppendLine(" [{0}] {1}".FormatInvariant(e.Span.ToDebugString(), e.Message)); } sb.AppendLine(); } if (errors.Warnings.Any()) { sb.AppendLine("Warnings:"); foreach (var e in errors.Warnings) { sb.AppendLine(" [{0}] {1}".FormatInvariant(e.Span.ToDebugString(), e.Message)); } } return(sb.ToString()); }
public void Imported(IModuleContext context) { if (_scraped) { return; } _scraped = true; var interp = context as AstPythonInterpreter; var fact = interp?.Factory as AstPythonInterpreterFactory; if (fact == null) { return; } var code = LoadCachedCode(interp); bool needCache = code == null; if (needCache) { if (!File.Exists(fact.Configuration.InterpreterPath)) { return; } var args = GetScrapeArguments(fact); if (args == null) { return; } var ms = new MemoryStream(); code = ms; using (var sw = new StreamWriter(ms, Encoding.UTF8, 4096, true)) using (var proc = new ProcessHelper( fact.Configuration.InterpreterPath, args, fact.Configuration.PrefixPath )) { proc.StartInfo.StandardOutputEncoding = Encoding.UTF8; proc.OnOutputLine = sw.WriteLine; proc.OnErrorLine = s => fact.Log(TraceLevel.Error, "Scrape", s); fact.Log(TraceLevel.Info, "Scrape", proc.FileName, proc.Arguments); proc.Start(); var exitCode = proc.Wait(60000); if (exitCode == null) { proc.Kill(); fact.Log(TraceLevel.Error, "ScrapeTimeout", proc.FileName, proc.Arguments); return; } else if (exitCode != 0) { fact.Log(TraceLevel.Error, "Scrape", "ExitCode", exitCode); return; } } code?.Seek(0, SeekOrigin.Begin); } if (code == null) { return; } PythonAst ast; using (code) { var sink = new CollectingErrorSink(); using (var sr = new StreamReader(code, Encoding.UTF8, true, 4096, true)) { var parser = Parser.CreateParser(sr, fact.GetLanguageVersion(), new ParserOptions { ErrorSink = sink, StubFile = true }); ast = parser.ParseFile(); } ParseErrors = sink.Errors.Select(e => "{0} ({1}): {2}".FormatUI(_filePath ?? "(builtins)", e.Span, e.Message)).ToArray(); if (ParseErrors.Any()) { fact.Log(TraceLevel.Error, "Parse", _filePath ?? "(builtins)"); foreach (var e in ParseErrors) { fact.Log(TraceLevel.Error, "Parse", e); } } if (needCache) { // We know we created the stream, so it's safe to seek here code.Seek(0, SeekOrigin.Begin); SaveCachedCode(interp, code); } } if (KeepAst) { Ast = ast; } #if DEBUG if (!string.IsNullOrEmpty(_filePath)) { var cachePath = fact.GetCacheFilePath(_filePath); if (!string.IsNullOrEmpty(cachePath)) { Locations = new[] { new LocationInfo(cachePath, null, 1, 1) }; } } #endif var walker = PrepareWalker(interp, ast); lock (_members) { ast.Walk(walker); PostWalk(walker); } }
public void Imported(IModuleContext context) { if (_scraped) { return; } _scraped = true; var interp = context as AstPythonInterpreter; var fact = interp?.Factory as AstPythonInterpreterFactory; if (fact == null) { return; } var code = LoadCachedCode(interp); bool needCache = code == null; if (needCache) { if (!File.Exists(fact.Configuration.InterpreterPath)) { return; } var args = GetScrapeArguments(fact); if (args == null) { return; } using (var p = ProcessOutput.Run( fact.Configuration.InterpreterPath, args.ToArray(), fact.Configuration.PrefixPath, null, visible: false, redirector: null, outputEncoding: Encoding.UTF8, errorEncoding: Encoding.UTF8 )) { p.Wait(); if (p.ExitCode == 0) { var ms = new MemoryStream(); code = ms; using (var sw = new StreamWriter(ms, Encoding.UTF8, 4096, true)) { foreach (var line in p.StandardOutputLines) { sw.WriteLine(line); } } code.Seek(0, SeekOrigin.Begin); } else { fact.Log(TraceLevel.Error, "Scrape", p.Arguments); foreach (var e in p.StandardErrorLines) { fact.Log(TraceLevel.Error, "Scrape", Name, e); } var err = new List <string> { $"Error scraping {Name}", p.Arguments }; err.AddRange(p.StandardErrorLines); Debug.Fail(string.Join(Environment.NewLine, err)); ParseErrors = err; } } } if (code == null) { return; } PythonAst ast; using (code) { var sink = new CollectingErrorSink(); using (var sr = new StreamReader(code, Encoding.UTF8, true, 4096, true)) using (var parser = Parser.CreateParser(sr, fact.GetLanguageVersion(), new ParserOptions { ErrorSink = sink })) { ast = parser.ParseFile(); } ParseErrors = sink.Errors.Select(e => $"{_filePath ?? "(builtins)"} ({e.Span}): {e.Message}").ToArray(); if (ParseErrors.Any()) { fact.Log(TraceLevel.Error, "Parse", _filePath ?? "(builtins)"); foreach (var e in ParseErrors) { fact.Log(TraceLevel.Error, "Parse", e); } } if (needCache) { // We know we created the stream, so it's safe to seek here code.Seek(0, SeekOrigin.Begin); SaveCachedCode(interp, code); } } #if DEBUG Locations = new[] { new LocationInfo(fact.GetCacheFilePath(_filePath), 1, 1) }; #endif var walker = PrepareWalker(interp, ast); lock (_members) { ast.Walk(walker); PostWalk(walker); } }
public void Imported(IModuleContext context) { if (_scraped) { return; } _scraped = true; var interp = context as AstPythonInterpreter; var fact = interp?.Factory; if (fact == null) { return; } var code = LoadCachedCode(interp); bool needCache = code == null; if (needCache) { if (!File.Exists(fact.Configuration.InterpreterPath)) { return; } var args = GetScrapeArguments(fact); if (args == null) { return; } using (var p = ProcessOutput.RunHiddenAndCapture(fact.Configuration.InterpreterPath, args.ToArray())) { p.Wait(); if (p.ExitCode == 0) { var ms = new MemoryStream(); code = ms; using (var sw = new StreamWriter(ms, Encoding.UTF8, 4096, true)) { foreach (var line in p.StandardOutputLines) { sw.WriteLine(line); } } code.Seek(0, SeekOrigin.Begin); } else { using (var sw = new StringWriter()) { sw.WriteLine("Error scraping builtins."); foreach (var line in p.StandardErrorLines) { sw.WriteLine(line); } Debug.Fail(sw.ToString()); } } } } if (code == null) { return; } PythonAst ast; using (code) { var sink = new CollectingErrorSink(); using (var sr = new StreamReader(code, Encoding.UTF8, true, 4096, true)) using (var parser = Parser.CreateParser(sr, fact.GetLanguageVersion(), new ParserOptions { ErrorSink = sink })) { ast = parser.ParseFile(); } foreach (var err in sink.Errors) { Trace.TraceError($"{_filePath ?? "(builtins)"} ({err.Span}): {err.Message}"); } if (needCache) { // We know we created the stream, so it's safe to seek here code.Seek(0, SeekOrigin.Begin); SaveCachedCode(interp, code); } } var walker = PrepareWalker(interp, ast); lock (_members) { ast.Walk(walker); PostWalk(walker); } }
/// <summary> /// Attempts to parse the given text. Returns true if the text is a valid expression. Returns false if the text is not /// a valid expression and assigns the error messages produced to errorMsg. /// </summary> public bool TryParseText(string text, out string errorMsg) { #if NEEDS_UPDATING CollectingErrorSink errorSink = new CollectingErrorSink(); Parser parser = Parser.CreateParser(new StringReader(text), _debugger.LanguageVersion, new ParserOptions() { ErrorSink = errorSink }); var ast = parser.ParseSingleStatement(); if (errorSink.Errors.Count > 0) { StringBuilder msg = new StringBuilder(); foreach (var error in errorSink.Errors) { msg.Append(error.Message); msg.Append(Environment.NewLine); } errorMsg = msg.ToString(); return false; } #endif errorMsg = null; return true; }