private void UpdateErrorsAndWarnings(
IProjectEntry entry,
ITextSnapshot snapshot,
CollectingErrorSink errorSink,
List<TaskProviderItem> commentTasks
) {
// Update the warn-on-launch state for this entry
bool changed = false;
lock (_hasParseErrorsLock) {
changed = errorSink.Errors.Any() ? _hasParseErrors.Add(entry) : _hasParseErrors.Remove(entry);
}
if (changed) {
OnShouldWarnOnLaunchChanged(entry);
}
// Update the parser warnings/errors.
var factory = new TaskProviderItemFactory(snapshot);
if (errorSink.Warnings.Any() || errorSink.Errors.Any()) {
_errorProvider.ReplaceItems(
entry,
ParserTaskMoniker,
errorSink.Warnings
.Select(er => factory.FromErrorResult(_serviceProvider, er, VSTASKPRIORITY.TP_NORMAL, VSTASKCATEGORY.CAT_BUILDCOMPILE))
.Concat(errorSink.Errors.Select(er => factory.FromErrorResult(_serviceProvider, er, VSTASKPRIORITY.TP_HIGH, VSTASKCATEGORY.CAT_BUILDCOMPILE)))
.ToList()
);
} else {
_errorProvider.Clear(entry, ParserTaskMoniker);
}
// Update comment tasks.
if (commentTasks.Count != 0) {
_commentTaskProvider.ReplaceItems(entry, ParserTaskMoniker, commentTasks);
} else {
_commentTaskProvider.Clear(entry, ParserTaskMoniker);
}
}
internal Task Analyze() {
if (_updater != null) {
_updater.UpdateStatus(_progressOffset, _progressTotal, "Starting analysis");
}
if (!string.IsNullOrEmpty(_logDiagnostic) && AnalysisLog.Output == null) {
try {
AnalysisLog.Output = new StreamWriter(new FileStream(_logDiagnostic, FileMode.Create, FileAccess.Write, FileShare.Read), Encoding.UTF8);
AnalysisLog.AsCSV = _logDiagnostic.EndsWith(".csv", StringComparison.InvariantCultureIgnoreCase);
} catch (Exception ex) {
TraceWarning("Failed to open \"{0}\" for logging{1}{2}", _logDiagnostic, Environment.NewLine, ex.ToString());
}
}
var callDepthOverrides = GetCallDepthOverrides();
var skipModules = new HashSet<string>(GetSkipModules(), StringComparer.Ordinal);
foreach (var files in _analyzeFileGroups) {
if (_cancel.IsCancellationRequested) {
break;
}
if (files.Count == 0) {
continue;
}
var outDir = GetOutputDir(files[0]);
if (_dryRun) {
foreach (var file in files) {
if (ContainsModule(skipModules, file.ModuleName)) {
continue;
}
Debug.Assert(!file.IsCompiled);
var idbFile = PathUtils.CreateFriendlyDirectoryPath(
_outDir,
Path.Combine(outDir, file.ModuleName)
);
TraceDryRun("ANALYZE;{0};{1}.idb", file.SourceFile, idbFile);
}
continue;
}
Directory.CreateDirectory(outDir);
TraceInformation("Start group \"{0}\" with {1} files", files[0].LibraryPath, files.Count);
AnalysisLog.StartFileGroup(files[0].LibraryPath, files.Count);
Console.WriteLine("Now analyzing: {0}", files[0].LibraryPath);
string currentLibrary;
if (_treatPathsAsStandardLibrary.Contains(files[0].LibraryPath)) {
currentLibrary = "standard library";
} else {
currentLibrary = PathUtils.GetFileOrDirectoryName(files[0].LibraryPath);
}
using (var factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(
_version,
null,
new[] { _outDir, outDir }.Concat(_baseDb.Skip(1)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray()
))
using (var projectState = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions()) {
int? mostItemsInQueue = null;
if (_updater != null) {
projectState.SetQueueReporting(itemsInQueue => {
if (itemsInQueue > (mostItemsInQueue ?? 0)) {
mostItemsInQueue = itemsInQueue;
}
if (mostItemsInQueue > 0) {
var progress = (files.Count * (mostItemsInQueue - itemsInQueue)) / mostItemsInQueue;
_updater.UpdateStatus(_progressOffset + (progress ?? 0), _progressTotal,
"Analyzing " + currentLibrary);
} else {
_updater.UpdateStatus(_progressOffset + files.Count, _progressTotal,
"Analyzing " + currentLibrary);
}
}, 10);
}
try {
using (var key = Registry.CurrentUser.OpenSubKey(AnalysisLimitsKey)) {
projectState.Limits = AnalysisLimits.LoadFromStorage(key, defaultToStdLib: true);
}
} catch (SecurityException) {
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
} catch (UnauthorizedAccessException) {
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
} catch (IOException) {
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
}
var items = files.Select(f => new AnalysisItem(f)).ToList();
foreach (var item in items) {
if (_cancel.IsCancellationRequested) {
break;
}
item.Entry = projectState.AddModule(item.ModuleName, item.SourceFile);
foreach (var name in ModulePath.GetParents(item.ModuleName, includeFullName: true)) {
int depth;
if (callDepthOverrides.TryGetValue(name, out depth)) {
TraceVerbose("Set CallDepthLimit to 0 for {0}", item.ModuleName);
item.Entry.Properties[AnalysisLimits.CallDepthKey] = depth;
break;
}
}
}
foreach (var item in items) {
if (_cancel.IsCancellationRequested) {
break;
}
if (ContainsModule(skipModules, item.ModuleName)) {
continue;
}
if (_updater != null) {
_updater.UpdateStatus(_progressOffset, _progressTotal,
string.Format("Parsing {0}", currentLibrary));
}
try {
var sourceUnit = new FileStream(item.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read);
var errors = new CollectingErrorSink();
var opts = new ParserOptions() { BindReferences = true, ErrorSink = errors };
TraceInformation("Parsing \"{0}\" (\"{1}\")", item.ModuleName, item.SourceFile);
item.Tree = Parser.CreateParser(sourceUnit, _version.ToLanguageVersion(), opts).ParseFile();
if (errors.Errors.Any() || errors.Warnings.Any()) {
TraceWarning("File \"{0}\" contained parse errors", item.SourceFile);
TraceInformation(string.Join(Environment.NewLine, errors.Errors.Concat(errors.Warnings)
.Select(er => string.Format("{0} {1}", er.Span, er.Message))));
}
} catch (Exception ex) {
TraceError("Error parsing \"{0}\" \"{1}\"{2}{3}", item.ModuleName, item.SourceFile, Environment.NewLine, ex.ToString());
}
}
TraceInformation("Parsing complete");
foreach (var item in items) {
if (_cancel.IsCancellationRequested) {
break;
}
if (item.Tree != null) {
item.Entry.UpdateTree(item.Tree, null);
}
}
foreach (var item in items) {
if (_cancel.IsCancellationRequested) {
break;
}
try {
if (item.Tree != null) {
TraceInformation("Analyzing \"{0}\"", item.ModuleName);
item.Entry.Analyze(_cancel, true);
TraceVerbose("Analyzed \"{0}\"", item.SourceFile);
}
} catch (Exception ex) {
TraceError("Error analyzing \"{0}\" \"{1}\"{2}{3}", item.ModuleName, item.SourceFile, Environment.NewLine, ex.ToString());
}
}
if (items.Count > 0 && !_cancel.IsCancellationRequested) {
TraceInformation("Starting analysis of {0} modules", items.Count);
items[0].Entry.AnalysisGroup.AnalyzeQueuedEntries(_cancel);
TraceInformation("Analysis complete");
}
if (_cancel.IsCancellationRequested) {
break;
}
TraceInformation("Saving group \"{0}\"", files[0].LibraryPath);
if (_updater != null) {
_progressOffset += files.Count;
_updater.UpdateStatus(_progressOffset, _progressTotal, "Saving " + currentLibrary);
}
Directory.CreateDirectory(outDir);
new SaveAnalysis().Save(projectState, outDir);
TraceInformation("End of group \"{0}\"", files[0].LibraryPath);
AnalysisLog.EndFileGroup();
AnalysisLog.Flush();
}
}
// Lets us have an awaitable function, even though it doesn't need
// to be async yet. This helps keep the interfaces consistent.
return Task.FromResult<object>(null);
}
private void ParsePythonCode(
ITextSnapshot snapshot, TextReader content, Severity indentationSeverity,
out PythonAst ast, out CollectingErrorSink errorSink, out List<TaskProviderItem> commentTasks
) {
ast = null;
errorSink = new CollectingErrorSink();
var tasks = commentTasks = new List<TaskProviderItem>();
var options = new ParserOptions {
ErrorSink = errorSink,
IndentationInconsistencySeverity = indentationSeverity,
BindReferences = true,
};
options.ProcessComment += (sender, e) => ProcessComment(tasks, snapshot, e.Span, e.Text);
using (var parser = Parser.CreateParser(content, Project.LanguageVersion, options)) {
ast = ParseOneFile(ast, parser);
}
}
private void ParseFile(IProjectEntry entry, IDictionary<int, CodeInfo> buffers) {
IPythonProjectEntry pyEntry;
IExternalProjectEntry externalEntry;
SortedDictionary<int, ParseResult> parseResults = new SortedDictionary<int, ParseResult>();
if ((pyEntry = entry as IPythonProjectEntry) != null) {
foreach (var buffer in buffers) {
var errorSink = new CollectingErrorSink();
var tasks = new List<AP.TaskItem>();
ParserOptions options = MakeParserOptions(errorSink, tasks);
using (var parser = buffer.Value.CreateParser(Project.LanguageVersion, options)) {
var ast = ParseOneFile(parser);
parseResults[buffer.Key] = new ParseResult(
ast,
errorSink,
tasks,
buffer.Value.Version
);
}
}
// Save the single or combined tree into the project entry
UpdateAnalysisTree(pyEntry, parseResults);
// update squiggles for the buffer. snapshot may be null if we
// are analyzing a file that is not open
SendParseComplete(pyEntry, parseResults);
// enqueue analysis of the file
if (parseResults.Where(x => x.Value.Ast != null).Any()) {
_analysisQueue.Enqueue(pyEntry, AnalysisPriority.Normal);
}
} else if ((externalEntry = entry as IExternalProjectEntry) != null) {
foreach (var keyValue in buffers) {
externalEntry.ParseContent(keyValue.Value.GetReader(), null);
_analysisQueue.Enqueue(entry, AnalysisPriority.Normal);
}
}
}
private ParserOptions MakeParserOptions(CollectingErrorSink errorSink, List<AP.TaskItem> tasks) {
var options = new ParserOptions {
ErrorSink = errorSink,
IndentationInconsistencySeverity = _options.indentation_inconsistency_severity,
BindReferences = true
};
options.ProcessComment += (sender, e) => ProcessComment(tasks, e.Span, e.Text);
return options;
}
public ParseResult(PythonAst ast, CollectingErrorSink errors, List<AP.TaskItem> tasks, int version) {
Ast = ast;
Errors = errors;
Tasks = tasks;
Version = version;
}
private static PythonAst ParseFileNoErrors(string filename, PythonLanguageVersion version, Severity indentationInconsistencySeverity = Severity.Ignore) {
var errorSink = new CollectingErrorSink();
var ast = ParseFile(filename, errorSink, version, indentationInconsistencySeverity);
foreach (var warn in errorSink.Warnings) {
Trace.TraceInformation("WARN: {0} {1}", warn.Span, warn.Message);
}
foreach (var err in errorSink.Errors) {
Trace.TraceInformation("ERR: {0} {1}", err.Span, err.Message);
}
Assert.AreEqual(0, errorSink.Warnings.Count + errorSink.Errors.Count, "Parse errors occurred");
return ast;
}
private void ParseErrors(string filename, PythonLanguageVersion version, Severity indentationInconsistencySeverity, params ErrorInfo[] errors) {
var sink = new CollectingErrorSink();
ParseFile(filename, sink, version, indentationInconsistencySeverity);
StringBuilder foundErrors = new StringBuilder();
for (int i = 0; i < sink.Errors.Count; i++) {
foundErrors.AppendFormat("new ErrorInfo(\"{0}\", {1}, {2}, {3}, {4}, {5}, {6})," + Environment.NewLine,
sink.Errors[i].Message,
sink.Errors[i].Span.Start.Index,
sink.Errors[i].Span.Start.Line,
sink.Errors[i].Span.Start.Column,
sink.Errors[i].Span.End.Index,
sink.Errors[i].Span.End.Line,
sink.Errors[i].Span.End.Column
);
}
string finalErrors = foundErrors.ToString();
Console.WriteLine(finalErrors);
Assert.AreEqual(errors.Length, sink.Errors.Count, "Version: " + version + Environment.NewLine + "Unexpected errors: " + Environment.NewLine + finalErrors);
for (int i = 0; i < errors.Length; i++) {
if (sink.Errors[i].Message != errors[i].Message) {
Assert.Fail("Wrong msg for error {0}: expected {1}, got {2}", i, errors[i].Message, sink.Errors[i].Message);
}
if (sink.Errors[i].Span != errors[i].Span) {
Assert.Fail("Wrong span for error {0}: expected ({1}, {2}, {3} - {4}, {5}, {6}), got ({7}, {8}, {9}, {10}, {11}, {12})",
i,
errors[i].Span.Start.Index,
errors[i].Span.Start.Line,
errors[i].Span.Start.Column,
errors[i].Span.End.Index,
errors[i].Span.End.Line,
errors[i].Span.End.Column,
sink.Errors[i].Span.Start.Index,
sink.Errors[i].Span.Start.Line,
sink.Errors[i].Span.Start.Column,
sink.Errors[i].Span.End.Index,
sink.Errors[i].Span.End.Line,
sink.Errors[i].Span.End.Column
);
}
}
}
private static string StdLibWorker(PythonVersion curVersion) {
var files = new List<string>();
CollectFiles(curVersion.LibPath, files, new[] { "site-packages" });
var skippedFiles = new HashSet<string>(new[] {
"py3_test_grammar.py", // included in 2x distributions but includes 3x grammar
"py2_test_grammar.py", // included in 3x distributions but includes 2x grammar
"proxy_base.py", // included in Qt port to Py3k but installed in 2.x distributions
"test_pep3131.py" // we need to update to support this.
});
var errorSink = new CollectingErrorSink();
var errors = new Dictionary<string, List<ErrorResult>>();
foreach (var file in files) {
string filename = Path.GetFileName(file);
if (skippedFiles.Contains(filename) || filename.StartsWith("badsyntax_") || filename.StartsWith("bad_coding") || file.IndexOf("\\lib2to3\\tests\\") != -1) {
continue;
}
using (var parser = Parser.CreateParser(new StreamReader(file), curVersion.Version, new ParserOptions() { ErrorSink = errorSink })) {
var ast = parser.ParseFile();
}
if (errorSink.Errors.Count != 0) {
var fileErrors = errorSink.Errors.ToList();
if (curVersion.Configuration.Version == new Version(3, 5)) {
// TODO: https://github.com/Microsoft/PTVS/issues/337
fileErrors.RemoveAll(e => {
return e.Message == "non-keyword arg after keyword arg";
});
}
if (fileErrors.Any()) {
errors["\"" + file + "\""] = fileErrors;
errorSink = new CollectingErrorSink();
}
}
}
if (errors.Count != 0) {
StringBuilder errorList = new StringBuilder();
foreach (var keyValue in errors) {
errorList.Append(keyValue.Key + " :" + Environment.NewLine);
foreach (var error in keyValue.Value) {
errorList.AppendFormat(" {0} {1}{2}", error.Span, error.Message, Environment.NewLine);
}
}
return errorList.ToString();
}
return null;
}
/// <summary>
/// Attempts to parse the given text. Returns true if the text is a valid expression. Returns false if the text is not
/// a valid expression and assigns the error messages produced to errorMsg.
/// </summary>
public virtual bool TryParseText(string text, out string errorMsg) {
CollectingErrorSink errorSink = new CollectingErrorSink();
Parser parser = Parser.CreateParser(new StringReader(text), _thread.Process.LanguageVersion, new ParserOptions() { ErrorSink = errorSink });
var ast = parser.ParseSingleStatement();
if (errorSink.Errors.Count > 0) {
StringBuilder msg = new StringBuilder();
foreach (var error in errorSink.Errors) {
msg.Append(error.Message);
msg.Append(Environment.NewLine);
}
errorMsg = msg.ToString();
return false;
}
errorMsg = null;
return true;
}