private void ParsePythonCode(
ITextSnapshot snapshot, TextReader content, Severity indentationSeverity,
out PythonAst ast, out CollectingErrorSink errorSink, out List<TaskProviderItem> commentTasks
) {
ast = null;
errorSink = new CollectingErrorSink();
var tasks = commentTasks = new List<TaskProviderItem>();
var options = new ParserOptions {
ErrorSink = errorSink,
IndentationInconsistencySeverity = indentationSeverity,
BindReferences = true,
};
options.ProcessComment += (sender, e) => ProcessComment(tasks, snapshot, e.Span, e.Text);
using (var parser = Parser.CreateParser(content, Project.LanguageVersion, options)) {
ast = ParseOneFile(ast, parser);
}
}
public unsafe void SetValueAsString(DkmEvaluationResult result, string value, int timeout, out string errorText) {
var pyEvalResult = result.GetDataItem<PyObjectEvaluationResult>();
if (pyEvalResult == null) {
Debug.Fail("SetValueAsString called on a DkmEvaluationResult without an associated PyObjectEvaluationResult.");
throw new NotSupportedException();
}
var proxy = pyEvalResult.ValueStore as IWritableDataProxy;
if (proxy == null) {
Debug.Fail("SetValueAsString called on a DkmEvaluationResult that does not correspond to an IWritableDataProxy.");
throw new InvalidOperationException();
}
errorText = null;
var process = result.StackFrame.Process;
var pyrtInfo = process.GetPythonRuntimeInfo();
var parserOptions = new ParserOptions { ErrorSink = new StringErrorSink() };
var parser = Parser.CreateParser(new StringReader(value), pyrtInfo.LanguageVersion, parserOptions);
var body = (ReturnStatement)parser.ParseTopExpression().Body;
errorText = parserOptions.ErrorSink.ToString();
if (!string.IsNullOrEmpty(errorText)) {
return;
}
var expr = body.Expression;
while (true) {
var parenExpr = expr as ParenthesisExpression;
if (parenExpr == null) {
break;
}
expr = parenExpr.Expression;
}
int sign;
expr = ForceExplicitSign(expr, out sign);
PyObject newObj = null;
var constExpr = expr as ConstantExpression;
if (constExpr != null) {
if (constExpr.Value == null) {
newObj = PyObject.None(process);
} else if (constExpr.Value is bool) {
// In 2.7, 'True' and 'False' are reported as identifiers, not literals, and are handled separately below.
newObj = PyBoolObject33.Create(process, (bool)constExpr.Value);
} else if (constExpr.Value is string) {
if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27) {
newObj = PyUnicodeObject27.Create(process, (string)constExpr.Value);
} else {
newObj = PyUnicodeObject33.Create(process, (string)constExpr.Value);
}
} else if (constExpr.Value is AsciiString) {
newObj = PyBytesObject.Create(process, (AsciiString)constExpr.Value);
}
} else {
var unaryExpr = expr as UnaryExpression;
if (unaryExpr != null && sign != 0) {
constExpr = unaryExpr.Expression as ConstantExpression;
if (constExpr != null) {
if (constExpr.Value is BigInteger) {
newObj = PyLongObject.Create(process, (BigInteger)constExpr.Value * sign);
} else if (constExpr.Value is int) {
if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27) {
newObj = PyIntObject.Create(process, (int)constExpr.Value * sign);
} else {
newObj = PyLongObject.Create(process, (int)constExpr.Value * sign);
}
} else if (constExpr.Value is double) {
newObj = PyFloatObject.Create(process, (double)constExpr.Value * sign);
} else if (constExpr.Value is Complex) {
newObj = PyComplexObject.Create(process, (Complex)constExpr.Value * sign);
}
}
} else {
var binExpr = expr as BinaryExpression;
if (binExpr != null && (binExpr.Operator == PythonOperator.Add || binExpr.Operator == PythonOperator.Subtract)) {
int realSign;
var realExpr = ForceExplicitSign(binExpr.Left, out realSign) as UnaryExpression;
int imagSign;
var imagExpr = ForceExplicitSign(binExpr.Right, out imagSign) as UnaryExpression;
if (realExpr != null && realSign != 0 && imagExpr != null && imagSign != 0) {
var realConst = realExpr.Expression as ConstantExpression;
var imagConst = imagExpr.Expression as ConstantExpression;
if (realConst != null && imagConst != null) {
var realVal = (realConst.Value as int? ?? realConst.Value as double?) as IConvertible;
var imagVal = imagConst.Value as Complex?;
if (realVal != null && imagVal != null) {
double real = realVal.ToDouble(null) * realSign;
double imag = imagVal.Value.Imaginary * imagSign * (binExpr.Operator == PythonOperator.Add ? 1 : -1);
newObj = PyComplexObject.Create(process, new Complex(real, imag));
}
}
}
} else {
if (pyrtInfo.LanguageVersion <= PythonLanguageVersion.V27) {
// 'True' and 'False' are not literals in 2.x, but we want to treat them as such.
var name = expr as NameExpression;
if (name != null) {
if (name.Name == "True") {
newObj = PyBoolObject27.Create(process, true);
} else if (name.Name == "False") {
newObj = PyBoolObject27.Create(process, false);
}
}
}
}
}
}
if (newObj != null) {
var oldObj = proxy.Read() as PyObject;
if (oldObj != null) {
// We can't free the original value without running some code in the process, and it may be holding heap locks.
// So don't decrement refcount now, but instead add it to the list of objects for TraceFunc to GC when it gets
// a chance to run next time.
_process.GetDataItem<PyObjectAllocator>().QueueForDecRef(oldObj);
}
newObj.ob_refcnt.Increment();
proxy.Write(newObj);
} else {
errorText = "Only boolean, numeric or string literals and None are supported.";
}
}
internal Task Analyze() {
if (_updater != null) {
_updater.UpdateStatus(_progressOffset, _progressTotal, "Starting analysis");
}
if (!string.IsNullOrEmpty(_logDiagnostic) && AnalysisLog.Output == null) {
try {
AnalysisLog.Output = new StreamWriter(new FileStream(_logDiagnostic, FileMode.Create, FileAccess.Write, FileShare.Read), Encoding.UTF8);
AnalysisLog.AsCSV = _logDiagnostic.EndsWith(".csv", StringComparison.InvariantCultureIgnoreCase);
} catch (Exception ex) {
TraceWarning("Failed to open \"{0}\" for logging{1}{2}", _logDiagnostic, Environment.NewLine, ex.ToString());
}
}
var callDepthOverrides = GetCallDepthOverrides();
var skipModules = new HashSet<string>(GetSkipModules(), StringComparer.Ordinal);
foreach (var files in _analyzeFileGroups) {
if (_cancel.IsCancellationRequested) {
break;
}
if (files.Count == 0) {
continue;
}
var outDir = GetOutputDir(files[0]);
if (_dryRun) {
foreach (var file in files) {
if (ContainsModule(skipModules, file.ModuleName)) {
continue;
}
Debug.Assert(!file.IsCompiled);
var idbFile = PathUtils.CreateFriendlyDirectoryPath(
_outDir,
Path.Combine(outDir, file.ModuleName)
);
TraceDryRun("ANALYZE;{0};{1}.idb", file.SourceFile, idbFile);
}
continue;
}
Directory.CreateDirectory(outDir);
TraceInformation("Start group \"{0}\" with {1} files", files[0].LibraryPath, files.Count);
AnalysisLog.StartFileGroup(files[0].LibraryPath, files.Count);
Console.WriteLine("Now analyzing: {0}", files[0].LibraryPath);
string currentLibrary;
if (_treatPathsAsStandardLibrary.Contains(files[0].LibraryPath)) {
currentLibrary = "standard library";
} else {
currentLibrary = PathUtils.GetFileOrDirectoryName(files[0].LibraryPath);
}
using (var factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(
_version,
null,
new[] { _outDir, outDir }.Concat(_baseDb.Skip(1)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray()
))
using (var projectState = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions()) {
int? mostItemsInQueue = null;
if (_updater != null) {
projectState.SetQueueReporting(itemsInQueue => {
if (itemsInQueue > (mostItemsInQueue ?? 0)) {
mostItemsInQueue = itemsInQueue;
}
if (mostItemsInQueue > 0) {
var progress = (files.Count * (mostItemsInQueue - itemsInQueue)) / mostItemsInQueue;
_updater.UpdateStatus(_progressOffset + (progress ?? 0), _progressTotal,
"Analyzing " + currentLibrary);
} else {
_updater.UpdateStatus(_progressOffset + files.Count, _progressTotal,
"Analyzing " + currentLibrary);
}
}, 10);
}
try {
using (var key = Registry.CurrentUser.OpenSubKey(AnalysisLimitsKey)) {
projectState.Limits = AnalysisLimits.LoadFromStorage(key, defaultToStdLib: true);
}
} catch (SecurityException) {
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
} catch (UnauthorizedAccessException) {
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
} catch (IOException) {
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
}
var items = files.Select(f => new AnalysisItem(f)).ToList();
foreach (var item in items) {
if (_cancel.IsCancellationRequested) {
break;
}
item.Entry = projectState.AddModule(item.ModuleName, item.SourceFile);
foreach (var name in ModulePath.GetParents(item.ModuleName, includeFullName: true)) {
int depth;
if (callDepthOverrides.TryGetValue(name, out depth)) {
TraceVerbose("Set CallDepthLimit to 0 for {0}", item.ModuleName);
item.Entry.Properties[AnalysisLimits.CallDepthKey] = depth;
break;
}
}
}
foreach (var item in items) {
if (_cancel.IsCancellationRequested) {
break;
}
if (ContainsModule(skipModules, item.ModuleName)) {
continue;
}
if (_updater != null) {
_updater.UpdateStatus(_progressOffset, _progressTotal,
string.Format("Parsing {0}", currentLibrary));
}
try {
var sourceUnit = new FileStream(item.SourceFile, FileMode.Open, FileAccess.Read, FileShare.Read);
var errors = new CollectingErrorSink();
var opts = new ParserOptions() { BindReferences = true, ErrorSink = errors };
TraceInformation("Parsing \"{0}\" (\"{1}\")", item.ModuleName, item.SourceFile);
item.Tree = Parser.CreateParser(sourceUnit, _version.ToLanguageVersion(), opts).ParseFile();
if (errors.Errors.Any() || errors.Warnings.Any()) {
TraceWarning("File \"{0}\" contained parse errors", item.SourceFile);
TraceInformation(string.Join(Environment.NewLine, errors.Errors.Concat(errors.Warnings)
.Select(er => string.Format("{0} {1}", er.Span, er.Message))));
}
} catch (Exception ex) {
TraceError("Error parsing \"{0}\" \"{1}\"{2}{3}", item.ModuleName, item.SourceFile, Environment.NewLine, ex.ToString());
}
}
TraceInformation("Parsing complete");
foreach (var item in items) {
if (_cancel.IsCancellationRequested) {
break;
}
if (item.Tree != null) {
item.Entry.UpdateTree(item.Tree, null);
}
}
foreach (var item in items) {
if (_cancel.IsCancellationRequested) {
break;
}
try {
if (item.Tree != null) {
TraceInformation("Analyzing \"{0}\"", item.ModuleName);
item.Entry.Analyze(_cancel, true);
TraceVerbose("Analyzed \"{0}\"", item.SourceFile);
}
} catch (Exception ex) {
TraceError("Error analyzing \"{0}\" \"{1}\"{2}{3}", item.ModuleName, item.SourceFile, Environment.NewLine, ex.ToString());
}
}
if (items.Count > 0 && !_cancel.IsCancellationRequested) {
TraceInformation("Starting analysis of {0} modules", items.Count);
items[0].Entry.AnalysisGroup.AnalyzeQueuedEntries(_cancel);
TraceInformation("Analysis complete");
}
if (_cancel.IsCancellationRequested) {
break;
}
TraceInformation("Saving group \"{0}\"", files[0].LibraryPath);
if (_updater != null) {
_progressOffset += files.Count;
_updater.UpdateStatus(_progressOffset, _progressTotal, "Saving " + currentLibrary);
}
Directory.CreateDirectory(outDir);
new SaveAnalysis().Save(projectState, outDir);
TraceInformation("End of group \"{0}\"", files[0].LibraryPath);
AnalysisLog.EndFileGroup();
AnalysisLog.Flush();
}
}
// Lets us have an awaitable function, even though it doesn't need
// to be async yet. This helps keep the interfaces consistent.
return Task.FromResult<object>(null);
}
/// <summary>
/// Gets the current AST and the code string for the project entry and returns the current code.
/// </summary>
public static PythonAst GetVerbatimAstAndCode(this IPythonProjectEntry projectFile, PythonLanguageVersion langVersion, int bufferId, out int version, out string code) {
ParserOptions options = new ParserOptions { BindReferences = true, Verbatim = true };
code = projectFile.GetCurrentCode(bufferId, out version);
if (code != null) {
var parser = Parser.CreateParser(
new StringReader(code),
langVersion,
options
);
return parser.ParseFile();
}
return null;
}
/// <summary>
/// Tries to evaluate the given expression by treating it as a chain of member access and indexing operations (e.g. <c>fob[0].oar.baz['abc'].blah</c>),
/// and looking up the corresponding members in data model provided by <see cref="GetFrameLocals"/>.
/// </summary>
/// <param name="vars">List of variables, in the context of which the expression is evaluated.</param>
/// <returns>
/// <c>true</c> if evaluation was successful, or if it failed and no fallback is possible (e.g. expression is invalid).
/// <c>false</c> if evaluation was not successful due to the limitations of this evaluator, and it may be possible to evaluate it correctly by other means.
/// </returns>
private bool EvaluateExpressionByWalkingObjects(IEnumerable<DkmSuccessEvaluationResult> vars, DkmInspectionContext inspectionContext, DkmWorkList workList, DkmLanguageExpression expression, DkmStackWalkFrame stackFrame, DkmCompletionRoutine<DkmEvaluateExpressionAsyncResult> completionRoutine) {
var pyrtInfo = stackFrame.Thread.Process.GetPythonRuntimeInfo();
var parserOptions = new ParserOptions { ErrorSink = new StringErrorSink() };
var parser = Parser.CreateParser(new StringReader(expression.Text), pyrtInfo.LanguageVersion, parserOptions);
var expr = ((ReturnStatement)parser.ParseTopExpression().Body).Expression;
string errorText = parserOptions.ErrorSink.ToString();
if (!string.IsNullOrEmpty(errorText)) {
completionRoutine(new DkmEvaluateExpressionAsyncResult(DkmFailedEvaluationResult.Create(
inspectionContext, stackFrame, expression.Text, expression.Text,
errorText, DkmEvaluationResultFlags.Invalid, null)));
return true;
}
// Unroll the AST into a sequence of member access and indexing operations, if possible.
var path = new Stack<string>();
var reprBuilder = new ReprBuilder(new ReprOptions(stackFrame.Thread.Process));
while (true) {
var memberExpr = expr as MemberExpression;
if (memberExpr != null) {
path.Push(memberExpr.Name);
expr = memberExpr.Target;
continue;
}
var indexingExpr = expr as IndexExpression;
if (indexingExpr != null) {
var indexExpr = indexingExpr.Index as ConstantExpression;
if (indexExpr != null) {
reprBuilder.Clear();
reprBuilder.AppendFormat("[{0:PY}]", indexExpr.Value);
path.Push(reprBuilder.ToString());
expr = indexingExpr.Target;
continue;
}
}
break;
}
var varExpr = expr as NameExpression;
if (varExpr == null) {
return false;
}
path.Push(varExpr.Name);
// Walk the path through Locals
while (true) {
var name = path.Pop();
var evalResult = vars.FirstOrDefault(er => er.Name == name);
if (evalResult == null) {
return false;
}
if (path.Count == 0) {
// Clone the evaluation result, but use expression text as its name.
DkmDataItem dataItem =
(DkmDataItem)evalResult.GetDataItem<PyObjectEvaluationResult>() ??
(DkmDataItem)evalResult.GetDataItem<GlobalsEvaluationResult>() ??
(DkmDataItem)evalResult.GetDataItem<CppViewEvaluationResult>() ??
(DkmDataItem)evalResult.GetDataItem<RawEvaluationResult>();
evalResult = DkmSuccessEvaluationResult.Create(
evalResult.InspectionContext,
evalResult.StackFrame,
expression.Text,
expression.Text,
evalResult.Flags,
evalResult.Value,
evalResult.EditableValue,
evalResult.Type,
evalResult.Category,
evalResult.Access,
evalResult.StorageType,
evalResult.TypeModifierFlags,
evalResult.Address,
evalResult.CustomUIVisualizers,
evalResult.ExternalModules,
dataItem);
completionRoutine(new DkmEvaluateExpressionAsyncResult(evalResult));
return true;
}
var childWorkList = DkmWorkList.Create(null);
evalResult.GetChildren(childWorkList, 0, inspectionContext, getChildrenResult =>
getChildrenResult.EnumContext.GetItems(childWorkList, 0, int.MaxValue, getItemsResult =>
vars = getItemsResult.Items.OfType<DkmSuccessEvaluationResult>()));
childWorkList.Execute();
}
}
public static Parser CreateParser(TextReader reader, PythonLanguageVersion version, ParserOptions parserOptions) {
if (reader == null) {
throw new ArgumentNullException("reader");
}
var options = parserOptions ?? ParserOptions.Default;
Parser parser = null;
var tokenizer = new Tokenizer(
version, options.ErrorSink,
(options.Verbatim ? TokenizerOptions.Verbatim : TokenizerOptions.None) | TokenizerOptions.GroupingRecovery,
(span, text) => options.RaiseProcessComment(parser, new CommentEventArgs(span, text)));
tokenizer.Initialize(null, reader, SourceLocation.MinValue);
tokenizer.IndentationInconsistencySeverity = options.IndentationInconsistencySeverity;
parser = new Parser(
tokenizer,
options.ErrorSink ?? ErrorSink.Null,
version,
options.Verbatim,
options.BindReferences,
options.PrivatePrefix
) { _sourceReader = reader };
return parser;
}
/// <summary>
/// Creates a new parser from a seekable stream including scanning the BOM or looking for a # coding: comment to detect the appropriate coding.
/// </summary>
public static Parser CreateParser(Stream stream, PythonLanguageVersion version, ParserOptions parserOptions = null) {
var options = parserOptions ?? ParserOptions.Default;
var defaultEncoding = version.Is2x() ? PythonAsciiEncoding.Instance : Encoding.UTF8;
var reader = GetStreamReaderWithEncoding(stream, defaultEncoding, options.ErrorSink);
return CreateParser(reader, version, options);
}
private ParserOptions MakeParserOptions(CollectingErrorSink errorSink, List<AP.TaskItem> tasks) {
var options = new ParserOptions {
ErrorSink = errorSink,
IndentationInconsistencySeverity = _options.indentation_inconsistency_severity,
BindReferences = true
};
options.ProcessComment += (sender, e) => ProcessComment(tasks, e.Span, e.Text);
return options;
}
public override Parser CreateParser(PythonLanguageVersion version, ParserOptions options) {
return Parser.CreateParser(_text, version, options);
}
public abstract Parser CreateParser(PythonLanguageVersion version, ParserOptions options);