public IMember GetValueFromExpression(Expression expr, LookupOptions options)
{
if (expr is ParenthesisExpression parExpr)
{
expr = parExpr.Expression;
}
if (expr == null)
{
return(null);
}
var m = GetValueFromName(expr as NameExpression, options) ??
GetValueFromMember(expr as MemberExpression, options) ??
GetValueFromCallable(expr as CallExpression, options) ??
GetValueFromUnaryOp(expr as UnaryExpression, options) ??
GetValueFromBinaryOp(expr, options) ??
GetValueFromIndex(expr as IndexExpression, options) ??
GetValueFromConditional(expr as ConditionalExpression, options) ??
GetConstantFromLiteral(expr, options);
if (m != null)
{
return(m);
}
_log?.Log(TraceLevel.Verbose, "UnknownExpression", expr.ToCodeString(Ast).Trim());
return(null);
}
public void LogFileRotation()
{
var logFile = Path.Combine(TestData.GetTempPath(), "Log.txt");
var writer = new AnalysisLogWriter(logFile, false, false);
for (int i = 0; i < 100; ++i)
{
writer.Log("Event", i);
}
writer.Flush(synchronous: true);
var lines = File.ReadAllLines(logFile);
Assert.AreEqual(101, lines.Length);
writer.Rotate(11);
lines = File.ReadAllLines(logFile);
AssertUtil.ContainsExactly(lines.Select(l => l.Substring(l.IndexOf(']') + 1).Trim()),
"Event: 90",
"Event: 91",
"Event: 92",
"Event: 93",
"Event: 94",
"Event: 95",
"Event: 96",
"Event: 97",
"Event: 98",
"Event: 99"
);
}
internal void Log(TraceLevel level, string eventName, params object[] args)
{
if (level <= _logLevel)
{
_log?.Log(eventName, args);
}
}
public override bool Walk(AssignmentStatement node)
{
var value = _scope.GetValueFromExpression(node.Right);
if ((value == null || value.MemberType == PythonMemberType.Unknown) && WarnAboutUndefinedValues)
{
_log?.Log(TraceLevel.Warning, "UndefinedValue", node.Right.ToCodeString(_ast).Trim());
}
foreach (var ne in node.Left.OfType <NameExpression>())
{
_scope.SetInScope(ne.Name, Clone(value));
}
return(base.Walk(node));
}
public async Task <IPythonModule> ImportModuleAsync(string name, CancellationToken token)
{
if (name == BuiltinModuleName)
{
if (_builtinModule == null)
{
_modules[BuiltinModuleName] = _builtinModule = new AstBuiltinsPythonModule(_factory.LanguageVersion);
_builtinModuleNames = null;
}
return(_builtinModule);
}
var ctxt = new AstPythonInterpreterFactory.TryImportModuleContext {
Interpreter = this,
ModuleCache = _modules,
BuiltinModule = _builtinModule,
FindModuleInUserSearchPathAsync = FindModuleInUserSearchPathAsync,
TypeStubPaths = _analyzer.Limits.UseTypeStubPackages ? _analyzer.GetTypeStubPaths() : null,
MergeTypeStubPackages = !_analyzer.Limits.UseTypeStubPackagesExclusively
};
for (var retries = 5; retries > 0; --retries)
{
// The call should be cancelled by the cancellation token, but since we
// are blocking here we wait for slightly longer. Timeouts are handled
// gracefully by TryImportModuleAsync(), so we want those to trigger if
// possible, but if all else fails then we'll abort and treat it as an
// error.
// (And if we've got a debugger attached, don't time out at all.)
AstPythonInterpreterFactory.TryImportModuleResult result;
try {
result = await _factory.TryImportModuleAsync(name, ctxt, token);
} catch (OperationCanceledException) {
_log.Log(TraceLevel.Error, "ImportTimeout", name);
Debug.Fail("Import timeout");
return(null);
}
switch (result.Status)
{
case AstPythonInterpreterFactory.TryImportModuleResultCode.Success:
return(result.Module);
case AstPythonInterpreterFactory.TryImportModuleResultCode.ModuleNotFound:
_log?.Log(TraceLevel.Info, "ImportNotFound", name);
return(null);
case AstPythonInterpreterFactory.TryImportModuleResultCode.NeedRetry:
case AstPythonInterpreterFactory.TryImportModuleResultCode.Timeout:
break;
case AstPythonInterpreterFactory.TryImportModuleResultCode.NotSupported:
_log?.Log(TraceLevel.Error, "ImportNotSupported", name);
return(null);
}
}
// Never succeeded, so just log the error and fail
_log?.Log(TraceLevel.Error, "RetryImport", name);
return(null);
}
public string GetCacheFilePath(string filePath)
{
if (!PathEqualityComparer.IsValidPath(DatabasePath))
{
if (!_loggedBadDbPath)
{
_loggedBadDbPath = true;
_log?.Log(TraceLevel.Warning, "InvalidDatabasePath", DatabasePath);
}
return(null);
}
var name = PathUtils.GetFileName(filePath);
if (!PathEqualityComparer.IsValidPath(name))
{
_log?.Log(TraceLevel.Warning, "InvalidCacheName", name);
return(null);
}
try {
var candidate = Path.ChangeExtension(Path.Combine(DatabasePath, name), ".pyi");
if (File.Exists(candidate))
{
return(candidate);
}
} catch (ArgumentException) {
return(null);
}
var hash = SHA256.Create();
var dir = Path.GetDirectoryName(filePath);
if (IsWindows())
{
dir = dir.ToLowerInvariant();
}
var dirHash = Convert.ToBase64String(hash.ComputeHash(new UTF8Encoding(false).GetBytes(dir)))
.Replace('/', '_').Replace('+', '-');
return(Path.ChangeExtension(Path.Combine(
DatabasePath,
Path.Combine(dirHash, name)
), ".pyi"));
}
private async Task <IReadOnlyDictionary <string, string> > GetUserSearchPathPackagesAsync()
{
_log?.Log(TraceLevel.Verbose, "GetUserSearchPathPackagesAsync");
var ussp = _userSearchPathPackages;
if (ussp == null)
{
IReadOnlyList <string> usp;
lock (_userSearchPathsLock) {
usp = _userSearchPaths;
ussp = _userSearchPathPackages;
}
if (ussp != null || usp == null || !usp.Any())
{
return(ussp);
}
_log?.Log(TraceLevel.Verbose, "GetImportableModulesAsync");
ussp = await AstPythonInterpreterFactory.GetImportableModulesAsync(usp);
lock (_userSearchPathsLock) {
if (_userSearchPathPackages == null)
{
_userSearchPathPackages = ussp;
}
else
{
ussp = _userSearchPathPackages;
}
}
}
_log?.Log(TraceLevel.Verbose, "GetUserSearchPathPackagesAsync", ussp.Keys.Cast <object>().ToArray());
return(ussp);
}
public override bool Walk(AssignmentStatement node)
{
var value = _scope.GetValueFromExpression(node.Right);
if ((value == null || value.MemberType == PythonMemberType.Unknown) && WarnAboutUndefinedValues)
{
_log?.Log(TraceLevel.Warning, "UndefinedValue", node.Right.ToCodeString(_ast).Trim());
}
if ((value as IPythonConstant)?.Type?.TypeId == BuiltinTypeId.Ellipsis)
{
value = _unknownType;
}
foreach (var expr in node.Left.OfType <ExpressionWithAnnotation>())
{
if (expr.Expression is NameExpression ne)
{
var annType = _scope.GetValueFromExpression(expr.Annotation) as IPythonType;
if (annType != null)
{
_scope.SetInScope(ne.Name, new AstPythonConstant(annType, GetLoc(expr.Expression)));
}
else
{
_scope.SetInScope(ne.Name, _unknownType);
}
}
}
foreach (var ne in node.Left.OfType <NameExpression>())
{
_scope.SetInScope(ne.Name, Clone(value));
}
return(base.Walk(node));
}
public IPythonModule ImportModule(string name)
{
IPythonModule module = null;
bool needRetry = true;
for (int retries = 5; retries > 0 && needRetry; --retries)
{
module = ImportModuleOrRetry(name, out needRetry);
}
if (needRetry)
{
// Never succeeded, so just log the error and fail
_log?.Log(TraceLevel.Error, "RetryImport", name);
return(null);
}
return(module);
}
public IMember GetValueFromExpression(Expression expr, LookupOptions options)
{
if (expr is ParenthesisExpression parExpr)
{
expr = parExpr.Expression;
}
if (expr == null)
{
return(null);
}
IMember m;
switch (expr)
{
case NameExpression nex:
m = GetValueFromName(nex, options);
break;
case MemberExpression mex:
m = GetValueFromMember(mex, options);
break;
case CallExpression cex:
m = GetValueFromCallable(cex, options);
break;
case UnaryExpression uex:
m = GetValueFromUnaryOp(uex, options);
break;
case IndexExpression iex:
m = GetValueFromIndex(iex, options);
break;
case ConditionalExpression coex:
m = GetValueFromConditional(coex, options);
break;
default:
m = GetValueFromBinaryOp(expr, options) ?? GetConstantFromLiteral(expr, options);
break;
}
if (m == null)
{
_log?.Log(TraceLevel.Verbose, "UnknownExpression", expr.ToCodeString(Ast).Trim());
}
return(m);
}
private async Task <IReadOnlyDictionary <string, string> > GetUserSearchPathPackagesAsync(CancellationToken cancellationToken)
{
_log?.Log(TraceLevel.Verbose, "GetUserSearchPathPackagesAsync");
var ussp = _userSearchPathPackages;
if (ussp == null)
{
IReadOnlyList <string> usp;
lock (_userSearchPathsLock) {
usp = _userSearchPaths;
ussp = _userSearchPathPackages;
}
if (ussp != null || usp == null || !usp.Any())
{
return(ussp);
}
_log?.Log(TraceLevel.Verbose, "GetImportableModulesAsync");
var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_factory.Configuration.Version);
ussp = await AstPythonInterpreterFactory.GetImportableModulesAsync(usp, requireInitPy, cancellationToken);
lock (_userSearchPathsLock) {
if (_userSearchPathPackages == null)
{
_userSearchPathPackages = ussp;
}
else
{
ussp = _userSearchPathPackages;
}
}
}
_log?.Log(TraceLevel.Verbose, "GetUserSearchPathPackagesAsync", ussp.Keys.Cast <object>().ToArray());
return(ussp);
}
public async Task <IReadOnlyList <string> > GetSearchPathsAsync(CancellationToken cancellationToken)
{
if (_searchPaths != null)
{
return(_searchPaths);
}
_searchPaths = await GetCurrentSearchPathsAsync(cancellationToken).ConfigureAwait(false);
Debug.Assert(_searchPaths != null, "Should have search paths");
_log?.Log(TraceLevel.Info, "SearchPaths", _searchPaths.Cast <object>().ToArray());
return(_searchPaths);
}
