private SaveLoadResult SaveLoad(PythonLanguageVersion version, params AnalysisModule[] modules) {
IPythonProjectEntry[] entries = new IPythonProjectEntry[modules.Length];
var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion());
var interp = fact.CreateInterpreter();
var dbFolder = TestData.GetTempPath();
var state = new PythonAnalysis(fact, interp, SharedDatabaseState.BuiltinName2x);
state.CreateProjectOnDisk = true;
for (int i = 0; i < modules.Length; i++) {
state.AddModule(modules[i].ModuleName, modules[i].Code, modules[i].Filename);
}
state.WaitForAnalysis();
new SaveAnalysis().Save(state.Analyzer, dbFolder);
File.Copy(
Path.Combine(PythonTypeDatabase.BaselineDatabasePath, "__builtin__.idb"),
Path.Combine(dbFolder, "__builtin__.idb"),
true
);
var loadFactory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(
version.ToVersion(),
null,
dbFolder
);
return new SaveLoadResult(CreateAnalyzer(loadFactory), state.CodeFolder);
}
private SaveLoadResult SaveLoad(PythonLanguageVersion version, params AnalysisModule[] modules)
{
IPythonProjectEntry[] entries = new IPythonProjectEntry[modules.Length];
var fact = PythonInterpreterFactoryWithDatabase.CreateFromDatabase(version.ToVersion());
var interp = fact.CreateInterpreter();
var dbFolder = TestData.GetTempPath();
var state = new PythonAnalysis(fact, interp);
state.CreateProjectOnDisk = true;
for (int i = 0; i < modules.Length; i++)
{
state.AddModule(modules[i].ModuleName, modules[i].Code, modules[i].Filename);
}
state.WaitForAnalysis();
new SaveAnalysis().Save(state.Analyzer, dbFolder);
File.WriteAllText(Path.Combine(dbFolder, "database.ver"), PythonTypeDatabase.CurrentVersion.ToString());
File.Copy(
Path.Combine(PythonTypeDatabase.BaselineDatabasePath, state.Analyzer._builtinName + ".idb"),
Path.Combine(dbFolder, state.Analyzer._builtinName + ".idb"),
true
);
var loadFactory = PythonInterpreterFactoryWithDatabase.CreateFromDatabase(version.ToVersion(), dbFolder);
return(new SaveLoadResult(CreateAnalyzer(loadFactory), state.CodeFolder));
}
private static void FullStdLibTest(PythonVersion v)
{
v.AssertInstalled();
var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList();
var paths = modules.Select(m => m.LibraryPath).Distinct().ToArray();
bool anySuccess = false;
using (var analyzer = new PythonAnalysis(v.Version)) {
analyzer.SetSearchPaths(paths);
foreach (var modName in modules)
{
if (modName.IsCompiled || modName.IsNativeExtension)
{
continue;
}
var mod = analyzer.Analyzer.Interpreter.ImportModule(modName.ModuleName);
if (mod == null)
{
Trace.TraceWarning("failed to import {0} from {1}".FormatInvariant(modName.ModuleName, modName.SourceFile));
}
else
{
anySuccess = true;
mod.GetMemberNames(analyzer.ModuleContext).ToList();
}
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
}
public void AstValues()
{
using (var entry = new PythonAnalysis(PythonLanguageVersion.V35)) {
entry.SetSearchPaths(TestData.GetPath(@"TestData\AstAnalysis"));
entry.AddModule("test-module", "from Values import *");
entry.WaitForAnalysis();
entry.AssertHasAttr("",
"x", "y", "z", "pi", "l", "t", "d", "s",
"X", "Y", "Z", "PI", "L", "T", "D", "S"
);
entry.AssertIsInstance("x", BuiltinTypeId.Int);
entry.AssertIsInstance("y", BuiltinTypeId.Str);
entry.AssertIsInstance("z", BuiltinTypeId.Bytes);
entry.AssertIsInstance("pi", BuiltinTypeId.Float);
entry.AssertIsInstance("l", BuiltinTypeId.List);
entry.AssertIsInstance("t", BuiltinTypeId.Tuple);
entry.AssertIsInstance("d", BuiltinTypeId.Dict);
entry.AssertIsInstance("s", BuiltinTypeId.Set);
entry.AssertIsInstance("X", BuiltinTypeId.Int);
entry.AssertIsInstance("Y", BuiltinTypeId.Str);
entry.AssertIsInstance("Z", BuiltinTypeId.Bytes);
entry.AssertIsInstance("PI", BuiltinTypeId.Float);
entry.AssertIsInstance("L", BuiltinTypeId.List);
entry.AssertIsInstance("T", BuiltinTypeId.Tuple);
entry.AssertIsInstance("D", BuiltinTypeId.Dict);
entry.AssertIsInstance("S", BuiltinTypeId.Set);
}
}
private static void FullStdLibTest(PythonVersion v)
{
v.AssertInstalled();
var factory = new AstPythonInterpreterFactory(v.Configuration, null);
var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList();
bool anySuccess = false;
bool anyExtensionSuccess = false, anyExtensionSeen = false;
using (var analyzer = new PythonAnalysis(factory)) {
foreach (var modName in modules)
{
anyExtensionSeen |= modName.IsNativeExtension;
var mod = analyzer.Analyzer.Interpreter.ImportModule(modName.ModuleName);
if (mod == null)
{
Trace.TraceWarning("failed to import {0} from {1}".FormatInvariant(modName.ModuleName, modName.SourceFile));
}
else
{
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
mod.GetMemberNames(analyzer.ModuleContext).ToList();
}
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules");
}
public void ImportPydFromSearchPath()
{
var analyzer = new PythonAnalysis("Global|PythonCore|2.7-32");
analyzer.AddModule("test-module", "from spam import *");
analyzer.WaitForAnalysis();
AssertUtil.CheckCollection(analyzer.GetAllNames(), null, new[] { "system", "spam" });
analyzer.SetSearchPaths(TestData.GetPath("TestData"));
analyzer.ReanalyzeAll(CancellationTokens.After60s);
AssertUtil.CheckCollection(analyzer.GetAllNames(), new[] { "system" }, new[] { "spam" });
}
public void ImportFromSearchPath()
{
var analyzer = new PythonAnalysis(PythonLanguageVersion.V35);
analyzer.AddModule("test-module", "from test_package import *");
analyzer.WaitForAnalysis();
AssertUtil.CheckCollection(analyzer.GetAllNames(), null, new[] { "package_method", "package_method_two", "test_package" });
analyzer.SetSearchPaths(TestData.GetPath("TestData\\AddImport"));
analyzer.ReanalyzeAll();
AssertUtil.CheckCollection(analyzer.GetAllNames(), new[] { "package_method", "package_method_two" }, new[] { "test_package" });
}
public void ImportPydFromSearchPath()
{
PythonTypeDatabase.ExtensionModuleLoader.AlwaysGenerateDb = true;
try {
var analyzer = new PythonAnalysis("Global|PythonCore|2.7-32");
analyzer.AddModule("test-module", "from spam import *");
analyzer.WaitForAnalysis();
AssertUtil.CheckCollection(analyzer.GetAllNames(), null, new[] { "system", "spam" });
analyzer.SetSearchPaths(TestData.GetPath("TestData"));
analyzer.ReanalyzeAll(CancellationTokens.After60s);
AssertUtil.CheckCollection(analyzer.GetAllNames(), new[] { "system" }, new[] { "spam" });
} finally {
PythonTypeDatabase.ExtensionModuleLoader.AlwaysGenerateDb = false;
}
}
private async Task FullStdLibTest(PythonVersion v, params string[] skipModules)
{
v.AssertInstalled();
var factory = new AstPythonInterpreterFactory(v.Configuration, new InterpreterFactoryCreationOptions {
DatabasePath = TestData.GetTempPath(),
UseExistingCache = false
});
var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList();
var skip = new HashSet <string>(skipModules);
skip.UnionWith(new[] {
"matplotlib.backends._backend_gdk",
"matplotlib.backends._backend_gtkagg",
"matplotlib.backends._gtkagg",
});
bool anySuccess = false;
bool anyExtensionSuccess = false, anyExtensionSeen = false;
bool anyParseError = false;
using (var analyzer = new PythonAnalysis(factory)) {
try {
var tasks = new List <Task <Tuple <ModulePath, IPythonModule> > >();
var interp = (AstPythonInterpreter)analyzer.Analyzer.Interpreter;
foreach (var m in skip)
{
interp.AddUnimportableModule(m);
}
foreach (var r in modules
.Where(m => !skip.Contains(m.ModuleName))
.GroupBy(m => {
int i = m.FullName.IndexOf('.');
return(i <= 0 ? m.FullName : m.FullName.Remove(i));
})
.AsParallel()
.SelectMany(g => g.Select(m => Tuple.Create(m, interp.ImportModule(m.ModuleName))))
)
{
var modName = r.Item1;
var mod = r.Item2;
anyExtensionSeen |= modName.IsNativeExtension;
if (mod == null)
{
Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile);
}
else if (mod is AstScrapedPythonModule smod)
{
if (smod.ParseErrors?.Any() ?? false)
{
anyParseError = true;
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
foreach (var e in smod.ParseErrors)
{
Trace.TraceError(e);
}
}
else
{
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
mod.GetMemberNames(analyzer.ModuleContext).ToList();
}
}
else if (mod is AstPythonModule)
{
// pass
}
else
{
Trace.TraceError("imported {0} as type {1}", modName.ModuleName, mod.GetType().FullName);
}
}
} finally {
_analysisLog = analyzer.GetLogContent(CultureInfo.InvariantCulture);
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules");
Assert.IsFalse(anyParseError, "parse errors occurred");
}
public SaveLoadResult(PythonAnalysis analyzer, string dir)
{
Analyzer = analyzer;
_dir = dir;
}
private static void FullStdLibTest(PythonVersion v) {
v.AssertInstalled();
var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList();
var paths = modules.Select(m => m.LibraryPath).Distinct().ToArray();
bool anySuccess = false;
using (var analyzer = new PythonAnalysis(v.Version)) {
analyzer.SetSearchPaths(paths);
foreach (var modName in modules) {
if (modName.IsCompiled || modName.IsNativeExtension) {
continue;
}
var mod = analyzer.Analyzer.Interpreter.ImportModule(modName.ModuleName);
if (mod == null) {
Trace.TraceWarning("failed to import {0} from {1}".FormatInvariant(modName.ModuleName, modName.SourceFile));
} else {
anySuccess = true;
mod.GetMemberNames(analyzer.ModuleContext).ToList();
}
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
}
public void AstValues() {
using (var entry = new PythonAnalysis(PythonLanguageVersion.V35)) {
entry.SetSearchPaths(TestData.GetPath(@"TestData\AstAnalysis"));
entry.AddModule("test-module", "from Values import *");
entry.WaitForAnalysis();
entry.AssertHasAttr("",
"x", "y", "z", "pi", "l", "t", "d", "s",
"X", "Y", "Z", "PI", "L", "T", "D", "S"
);
entry.AssertIsInstance("x", BuiltinTypeId.Int);
entry.AssertIsInstance("y", BuiltinTypeId.Str);
entry.AssertIsInstance("z", BuiltinTypeId.Bytes);
entry.AssertIsInstance("pi", BuiltinTypeId.Float);
entry.AssertIsInstance("l", BuiltinTypeId.List);
entry.AssertIsInstance("t", BuiltinTypeId.Tuple);
entry.AssertIsInstance("d", BuiltinTypeId.Dict);
entry.AssertIsInstance("s", BuiltinTypeId.Set);
entry.AssertIsInstance("X", BuiltinTypeId.Int);
entry.AssertIsInstance("Y", BuiltinTypeId.Str);
entry.AssertIsInstance("Z", BuiltinTypeId.Bytes);
entry.AssertIsInstance("PI", BuiltinTypeId.Float);
entry.AssertIsInstance("L", BuiltinTypeId.List);
entry.AssertIsInstance("T", BuiltinTypeId.Tuple);
entry.AssertIsInstance("D", BuiltinTypeId.Dict);
entry.AssertIsInstance("S", BuiltinTypeId.Set);
}
}
public void ImportFromSearchPath() {
var analyzer = new PythonAnalysis(PythonLanguageVersion.V35);
analyzer.AddModule("test-module", "from test_package import *");
analyzer.WaitForAnalysis();
AssertUtil.CheckCollection(analyzer.GetAllNames(), null, new[] { "package_method", "package_method_two", "test_package" });
analyzer.SetSearchPaths(TestData.GetPath("TestData\\AddImport"));
analyzer.ReanalyzeAll();
AssertUtil.CheckCollection(analyzer.GetAllNames(), new[] { "package_method", "package_method_two" }, new[] { "test_package" });
}
public void ImportPydFromSearchPath() {
PythonTypeDatabase.ExtensionModuleLoader.AlwaysGenerateDb = true;
try {
var analyzer = new PythonAnalysis("Global|PythonCore|2.7-32");
analyzer.AddModule("test-module", "from spam import *");
analyzer.WaitForAnalysis();
AssertUtil.CheckCollection(analyzer.GetAllNames(), null, new[] { "system", "spam" });
analyzer.SetSearchPaths(TestData.GetPath("TestData"));
analyzer.ReanalyzeAll(CancellationTokens.After60s);
AssertUtil.CheckCollection(analyzer.GetAllNames(), new[] { "system" }, new[] { "spam" });
} finally {
PythonTypeDatabase.ExtensionModuleLoader.AlwaysGenerateDb = false;
}
}
public SaveLoadResult(PythonAnalysis analyzer, string dir) {
Analyzer = analyzer;
_dir = dir;
}
private SaveLoadResult SaveLoad(PythonLanguageVersion version, params AnalysisModule[] modules) {
IPythonProjectEntry[] entries = new IPythonProjectEntry[modules.Length];
var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion());
var interp = fact.CreateInterpreter();
var dbFolder = TestData.GetTempPath(randomSubPath: true);
Directory.CreateDirectory(dbFolder);
var state = new PythonAnalysis(fact, interp, SharedDatabaseState.BuiltinName2x);
state.CreateProjectOnDisk = true;
for (int i = 0; i < modules.Length; i++) {
state.AddModule(modules[i].ModuleName, modules[i].Code, modules[i].Filename);
}
state.WaitForAnalysis();
new SaveAnalysis().Save(state.Analyzer, dbFolder);
File.Copy(
Path.Combine(PythonTypeDatabase.BaselineDatabasePath, "__builtin__.idb"),
Path.Combine(dbFolder, "__builtin__.idb"),
true
);
var loadFactory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(
version.ToVersion(),
null,
dbFolder
);
return new SaveLoadResult(CreateAnalyzer(loadFactory), state.CodeFolder);
}
