private async Task <IPythonModule> HandlePossibleImportAsync(ImportStatement node, PossibleModuleImport possibleModuleImport, CancellationToken cancellationToken)
{
var fullName = possibleModuleImport.PrecedingModuleFullName;
var module = await ModuleResolution.ImportModuleAsync(possibleModuleImport.PrecedingModuleFullName, cancellationToken);
if (module == null)
{
MakeUnresolvedImport(possibleModuleImport.PrecedingModuleFullName, node);
return(null);
}
var nameParts = possibleModuleImport.RemainingNameParts;
for (var i = 0; i < nameParts.Count; i++)
{
var namePart = nameParts[i];
var childModule = module.GetMember <IPythonModule>(namePart);
if (childModule == null)
{
var unresolvedModuleName = string.Join(".", nameParts.Take(i + 1).Prepend(fullName));
MakeUnresolvedImport(unresolvedModuleName, node);
return(null);
}
module = childModule;
}
return(module);
}
private async Task HandleModuleImportStarAsync(IPythonModule module, CancellationToken cancellationToken = default)
{
foreach (var memberName in module.GetMemberNames())
{
cancellationToken.ThrowIfCancellationRequested();
var member = module.GetMember(memberName);
if (member == null)
{
Log?.Log(TraceEventType.Verbose, $"Undefined import: {module.Name}, {memberName}");
}
else if (member.MemberType == PythonMemberType.Unknown)
{
Log?.Log(TraceEventType.Verbose, $"Unknown import: {module.Name}, {memberName}");
}
member = member ?? Eval.UnknownType;
if (member is IPythonModule m)
{
await ModuleResolution.ImportModuleAsync(m.Name, cancellationToken);
}
Eval.DeclareVariable(memberName, member, module.Location);
}
}
private void HandleModuleImportStar(PythonVariableModule variableModule, bool isImplicitPackage)
{
if (variableModule.Module == Module)
{
// from self import * won't define any new members
return;
}
// If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore.
var memberNames = isImplicitPackage
? variableModule.GetMemberNames()
: variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_"));
foreach (var memberName in memberNames)
{
var member = variableModule.GetMember(memberName);
if (member == null)
{
Log?.Log(TraceEventType.Verbose, $"Undefined import: {variableModule.Name}, {memberName}");
}
else if (member.MemberType == PythonMemberType.Unknown)
{
Log?.Log(TraceEventType.Verbose, $"Unknown import: {variableModule.Name}, {memberName}");
}
member = member ?? Eval.UnknownType;
if (member is IPythonModule m)
{
ModuleResolution.GetOrLoadModule(m.Name);
}
var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import);
}
}
private void HandleModuleImportStar(PythonVariableModule variableModule)
{
if (variableModule.Module == Module)
{
// from self import * won't define any new members
return;
}
foreach (var memberName in variableModule.GetMemberNames())
{
var member = variableModule.GetMember(memberName);
if (member == null)
{
Log?.Log(TraceEventType.Verbose, $"Undefined import: {variableModule.Name}, {memberName}");
}
else if (member.MemberType == PythonMemberType.Unknown)
{
Log?.Log(TraceEventType.Verbose, $"Unknown import: {variableModule.Name}, {memberName}");
}
member = member ?? Eval.UnknownType;
if (member is IPythonModule m)
{
ModuleResolution.GetOrLoadModule(m.Name);
}
Eval.DeclareVariable(memberName, member, VariableSource.Import, variableModule.Location);
}
}
/// <summary>
/// Determines value of the variable and declares it. Value depends if source module has submodule
/// that is named the same as the variable and/or it has internal variables named same as the submodule.
/// </summary>
/// <example>'from a.b import c' when 'c' is both submodule of 'b' and a variable declared inside 'b'.</example>
/// <param name="variableModule">Source module of the variable such as 'a.b' in 'from a.b import c as d'.</param>
/// <param name="memberName">Module member name such as 'c' in 'from a.b import c as d'.</param>
/// <param name="imports">Import search result.</param>
/// <param name="variableName">Name of the variable to declare, such as 'd' in 'from a.b import c as d'.</param>
/// <param name="importPosition">Position of the import statement.</param>
/// <param name="nameLocation">Location of the variable name expression.</param>
private void DeclareVariable(PythonVariableModule variableModule, string memberName, IImportSearchResult imports, string variableName, int importPosition, Node nameLocation)
{
// First try imports since child modules should win, i.e. in 'from a.b import c'
// 'c' should be a submodule if 'b' has one, even if 'b' also declares 'c = 1'.
var value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName);
// First try exported or child submodules.
value = value ?? variableModule.GetMember(memberName);
// Value may be variable or submodule. If it is variable, we need it in order to add reference.
var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
value = variable?.Value?.Equals(value) == true ? variable : value;
// If nothing is exported, variables are still accessible.
value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType;
// Do not allow imported variables to override local declarations
var canOverwrite = CanOverwriteVariable(variableName, importPosition, value);
// Do not declare references to '*'
var locationExpression = nameLocation is NameExpression nex && nex.Name == "*" ? null : nameLocation;
Eval.DeclareVariable(variableName, value, VariableSource.Import, locationExpression, canOverwrite);
// Make sure module is loaded and analyzed.
if (value is IPythonModule m)
{
ModuleResolution.GetOrLoadModule(m.Name);
}
}
private async Task ImportMembersFromPackageAsync(FromImportStatement node, PackageImport packageImport, CancellationToken cancellationToken = default)
{
var names = node.Names;
var asNames = node.AsNames;
if (names.Count == 1 && names[0].Name == "*")
{
// TODO: Need tracking of previous imports to determine possible imports for namespace package. For now import nothing
Eval.DeclareVariable("*", Eval.UnknownType, Eval.GetLoc(names[0]));
return;
}
for (var i = 0; i < names.Count; i++)
{
var importName = names[i].Name;
var memberReference = asNames[i] ?? names[i];
var memberName = memberReference.Name;
var location = Eval.GetLoc(memberReference);
ModuleImport moduleImport;
IPythonType member;
if ((moduleImport = packageImport.Modules.FirstOrDefault(mi => mi.Name.EqualsOrdinal(importName))) != null)
{
member = await ModuleResolution.ImportModuleAsync(moduleImport.FullName, cancellationToken);
}
else
{
member = Eval.UnknownType;
}
Eval.DeclareVariable(memberName, member, location);
}
}
private async Task ImportMembersFromModuleAsync(FromImportStatement node, string moduleName, CancellationToken cancellationToken = default)
{
var names = node.Names;
var asNames = node.AsNames;
var module = await ModuleResolution.ImportModuleAsync(moduleName, cancellationToken);
if (names.Count == 1 && names[0].Name == "*")
{
// TODO: warn this is not a good style per
// TODO: https://docs.python.org/3/faq/programming.html#what-are-the-best-practices-for-using-import-in-a-module
// TODO: warn this is invalid if not in the global scope.
await HandleModuleImportStarAsync(module, cancellationToken);
return;
}
for (var i = 0; i < names.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var memberReference = asNames[i] ?? names[i];
var memberName = memberReference.Name;
var type = module?.GetMember(memberReference.Name);
Eval.DeclareVariable(memberName, type, names[i]);
}
}
public async Task <IPythonModule> ImportModuleAsync(string name, CancellationToken token)
{
if (name == BuiltinModuleName)
{
if (_builtinModule == null)
{
_modules[BuiltinModuleName] = _builtinModule = new AstBuiltinsPythonModule(_factory.LanguageVersion);
}
return(_builtinModule);
}
Debug.Assert(_analyzer != null);
var pathResolver = _analyzer.CurrentPathResolver;
var typeStubPaths = _analyzer.Limits.UseTypeStubPackages ? _analyzer.GetTypeStubPaths() : null;
var mergeTypeStubPackages = !_analyzer.Limits.UseTypeStubPackagesExclusively;
for (var retries = 5; retries > 0; --retries)
{
// The call should be cancelled by the cancellation token, but since we
// are blocking here we wait for slightly longer. Timeouts are handled
// gracefully by TryImportModuleAsync(), so we want those to trigger if
// possible, but if all else fails then we'll abort and treat it as an
// error.
// (And if we've got a debugger attached, don't time out at all.)
TryImportModuleResult result;
try {
result = await ModuleResolution.TryImportModuleAsync(name, pathResolver, typeStubPaths, mergeTypeStubPackages, token);
} catch (OperationCanceledException) {
Log.Log(TraceLevel.Error, "ImportTimeout", name);
Debug.Fail("Import timeout");
return(null);
}
switch (result.Status)
{
case TryImportModuleResultCode.Success:
return(result.Module);
case TryImportModuleResultCode.ModuleNotFound:
Log?.Log(TraceLevel.Info, "ImportNotFound", name);
return(null);
case TryImportModuleResultCode.NeedRetry:
case TryImportModuleResultCode.Timeout:
break;
case TryImportModuleResultCode.NotSupported:
Log?.Log(TraceLevel.Error, "ImportNotSupported", name);
return(null);
}
}
// Never succeeded, so just log the error and fail
Log?.Log(TraceLevel.Error, "RetryImport", name);
return(null);
}
public void Returns_politely_when_no_method_found()
{
SingletonModulePool <INetModule> pool = new SingletonModulePool <INetModule>(Substitute.For <INetModule>(), true);
_moduleProvider.Register(pool);
ModuleResolution resolution = _moduleProvider.Check("unknown_method");
Assert.AreEqual(ModuleResolution.Unknown, resolution);
}
public void Method_resolution_is_not_case_sensitive()
{
SingletonModulePool <INetModule> pool = new SingletonModulePool <INetModule>(new NetModule(LimboLogs.Instance, Substitute.For <INetBridge>()), true);
_moduleProvider.Register(pool);
ModuleResolution resolution = _moduleProvider.Check("net_VeRsIoN");
Assert.AreEqual(ModuleResolution.Enabled, resolution);
}
public void Module_provider_will_recognize_disabled_modules()
{
JsonRpcConfig jsonRpcConfig = new JsonRpcConfig();
jsonRpcConfig.EnabledModules = new string[0];
_moduleProvider = new RpcModuleProvider(jsonRpcConfig, LimboLogs.Instance);
_moduleProvider.Register(new SingletonModulePool <IProofModule>(Substitute.For <IProofModule>(), false));
ModuleResolution resolution = _moduleProvider.Check("proof_call");
Assert.AreEqual(ModuleResolution.Disabled, resolution);
}
private async Task <IPythonModule> HandleImportAsync(ImportStatement node, ModuleImport moduleImport, CancellationToken cancellationToken)
{
var module = await ModuleResolution.ImportModuleAsync(moduleImport.FullName, cancellationToken);
if (module == null)
{
MakeUnresolvedImport(moduleImport.FullName, node);
return(null);
}
return(module);
}
private bool InstallList(ModuleResolution modules, RelationshipResolverOptions options)
{
if (toInstall.Any())
{
// actual magic happens here, we run the installer with our mod list
var module_installer = ModuleInstaller.GetInstance(manager.CurrentInstance, GUI.user);
module_installer.onReportModInstalled = OnModInstalled;
return(WasSuccessful(() => module_installer.InstallList(modules, options)));
}
return(true);
}
public void With_filter_can_reject(string regex, ModuleResolution expectedResult)
{
JsonRpcConfig config = new JsonRpcConfig();
_fileSystem.File.Exists(Arg.Any <string>()).Returns(true);
_fileSystem.File.ReadLines(Arg.Any <string>()).Returns(new[] { regex });
_moduleProvider = new RpcModuleProvider(_fileSystem, config, LimboLogs.Instance);
SingletonModulePool <INetModule> pool = new SingletonModulePool <INetModule>(new NetModule(LimboLogs.Instance, Substitute.For <INetBridge>()), true);
_moduleProvider.Register(pool);
ModuleResolution resolution = _moduleProvider.Check("net_version");
resolution.Should().Be(expectedResult);
}
public void Method_resolution_is_scoped_to_url_enabled_modules()
{
_moduleProvider.Register(new SingletonModulePool <INetRpcModule>(Substitute.For <INetRpcModule>(), true));
_moduleProvider.Register(new SingletonModulePool <IProofRpcModule>(Substitute.For <IProofRpcModule>(), true));
JsonRpcUrl url = new JsonRpcUrl("http", "127.0.0.1", 8888, RpcEndpoint.Http, new[] { "net" });
ModuleResolution inScopeResolution = _moduleProvider.Check("net_version", JsonRpcContext.Http(url));
Assert.AreEqual(ModuleResolution.Enabled, inScopeResolution);
ModuleResolution outOfScopeResolution = _moduleProvider.Check("proof_call", JsonRpcContext.Http(url));
Assert.AreEqual(ModuleResolution.Disabled, outOfScopeResolution);
ModuleResolution fallbackResolution = _moduleProvider.Check("proof_call", new JsonRpcContext(RpcEndpoint.Http));
Assert.AreEqual(ModuleResolution.Enabled, fallbackResolution);
}
private IMember GetValueFromImports(PythonVariableModule parentModule, IImportChildrenSource childrenSource, string memberName)
{
if (childrenSource == null || !childrenSource.TryGetChildImport(memberName, out var childImport))
{
return(Interpreter.UnknownType);
}
switch (childImport)
{
case ModuleImport moduleImport:
var module = ModuleResolution.GetOrLoadModule(moduleImport.FullName);
return(module != null?GetOrCreateVariableModule(module, parentModule, moduleImport.Name) : Interpreter.UnknownType);
case ImplicitPackageImport packageImport:
return(GetOrCreateVariableModule(packageImport.FullName, parentModule, memberName));
default:
return(Interpreter.UnknownType);
}
}
public void Initialize(PythonAnalyzer analyzer)
{
if (_analyzer != null)
{
return;
}
_analyzer = analyzer;
if (analyzer != null)
{
var interpreterPaths = ModuleResolution.GetSearchPathsAsync(CancellationToken.None).WaitAndUnwrapExceptions();
_analyzer.SetInterpreterPaths(interpreterPaths);
analyzer.SearchPathsChanged += Analyzer_SearchPathsChanged;
var bm = analyzer.BuiltinModule;
if (!string.IsNullOrEmpty(bm?.Name))
{
_modules[analyzer.BuiltinModule.Name] = analyzer.BuiltinModule.InterpreterModule;
}
}
}
public void InstallList(ModuleResolution modules, RelationshipResolverOptions options)
{
// We're about to install all our mods; so begin our transaction.
using (TransactionScope transaction = CkanTransaction.CreateTransactionScope())
{
var enumeratedMods = modules.Select((m, i) => new { Idx = i, Module = m });
foreach (var item in enumeratedMods)
{
var percentComplete = (item.Idx * 100) / modules.Count;
User.RaiseProgress(string.Format("Installing mod \"{0}\"", item.Module), percentComplete);
Install(item.Module);
}
User.RaiseProgress("Updating registry", 70);
registry_manager.Save(!options.without_enforce_consistency);
User.RaiseProgress("Commiting filesystem changes", 80);
transaction.Complete();
}
}
/// <summary>
/// For test use only
/// </summary>
internal void SetCurrentSearchPaths(IEnumerable <string> paths)
{
ModuleResolution.SetCurrentSearchPaths(paths);
ImportableModulesChanged?.Invoke(this, EventArgs.Empty);
}
public void NotifyImportableModulesChanged() => ModuleResolution.ReloadAsync().DoNotWait();
