private async void OnNewDatabaseAvailable(object sender, EventArgs e) {
var factory = _factory;
if (factory == null) {
// We have been disposed already, so ignore this event
return;
}
_typeDb = factory.GetCurrentDatabase();
List<ProjectReference> references = null;
lock (_referencesLock) {
references = _references != null ? _references.ToList() : null;
}
if (references != null) {
_typeDb = _typeDb.Clone();
foreach (var reference in references) {
string modName;
try {
modName = Path.GetFileNameWithoutExtension(reference.Name);
} catch (ArgumentException) {
continue;
}
try {
await _typeDb.LoadExtensionModuleAsync(modName, reference.Name);
} catch (Exception ex) {
try {
Directory.CreateDirectory(factory.DatabasePath);
} catch (IOException) {
} catch (UnauthorizedAccessException) {
}
if (Directory.Exists(factory.DatabasePath)) {
var analysisLog = Path.Combine(factory.DatabasePath, "AnalysisLog.txt");
for (int retries = 10; retries > 0; --retries) {
try {
File.AppendAllText(analysisLog, string.Format(
"Exception while loading extension module {0}{1}{2}{1}",
reference.Name,
Environment.NewLine,
ex
));
break;
} catch (Exception ex2) {
if (ex2.IsCriticalException()) {
throw;
}
}
}
}
}
}
}
var evt = ModuleNamesChanged;
if (evt != null) {
evt(this, EventArgs.Empty);
}
}
private async void OnNewDatabaseAvailable(object sender, EventArgs e) {
var factory = _factory;
if (factory == null) {
// We have been disposed already, so ignore this event
return;
}
_typeDb = factory.GetCurrentDatabase();
_searchPathDb = null;
_zipPackageCache = null;
ModuleNamesChanged?.Invoke(this, EventArgs.Empty);
}
private void OnNewDatabaseAvailable(object sender, EventArgs e) {
_typeDb = _factory.GetCurrentDatabase();
if (_references != null) {
_typeDb = _typeDb.Clone();
foreach (var reference in _references) {
string modName;
try {
modName = Path.GetFileNameWithoutExtension(reference.Name);
} catch (Exception) {
continue;
}
_typeDb.LoadExtensionModuleAsync(modName, reference.Name).Wait();
}
}
var evt = ModuleNamesChanged;
if (evt != null) {
evt(this, EventArgs.Empty);
}
}
public void Dispose() {
_typeDb = null;
var factory = _factory;
_factory = null;
if (factory != null) {
factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
}
}
public Task AddReferenceAsync(ProjectReference reference, CancellationToken cancellationToken = default(CancellationToken)) {
if (reference == null) {
return MakeExceptionTask(new ArgumentNullException("reference"));
}
bool cloneDb = false;
lock (_referencesLock) {
if (_references == null) {
_references = new HashSet<ProjectReference>();
cloneDb = true;
}
}
if (cloneDb && _typeDb != null) {
// If we needed to set _references, then we also need to clone
// _typeDb to avoid adding modules to the shared database.
_typeDb = _typeDb.Clone();
}
switch (reference.Kind) {
case ProjectReferenceKind.ExtensionModule:
lock (_referencesLock) {
_references.Add(reference);
}
string filename;
try {
filename = Path.GetFileNameWithoutExtension(reference.Name);
} catch (Exception e) {
return MakeExceptionTask(e);
}
if (_typeDb != null) {
return _typeDb.LoadExtensionModuleAsync(filename,
reference.Name,
cancellationToken).ContinueWith(RaiseModulesChanged);
}
break;
}
return Task.Factory.StartNew(EmptyTask);
}
public AnalysisOnlyInterpreterFactory(Version version, PythonTypeDatabase database, string description = null)
: base(GetConfiguration(version, database.DatabaseDirectory), CreationOptions) {
_actualDatabase = database;
}
public void Dispose() {
if (_typeDb != null) {
_typeDb = null;
}
if (_factory != null) {
_factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
_factory = null;
}
}
private async void OnNewDatabaseAvailable(object sender, EventArgs e)
{
var factory = _factory;
if (factory == null)
{
// We have been disposed already, so ignore this event
return;
}
_typeDb = factory.GetCurrentDatabase();
List <ProjectReference> references = null;
lock (_referencesLock) {
references = _references != null?_references.ToList() : null;
}
if (references != null)
{
_typeDb = _typeDb.Clone();
foreach (var reference in references)
{
string modName;
try {
modName = Path.GetFileNameWithoutExtension(reference.Name);
} catch (ArgumentException) {
continue;
}
try {
await _typeDb.LoadExtensionModuleAsync(modName, reference.Name);
} catch (Exception ex) {
try {
Directory.CreateDirectory(factory.DatabasePath);
} catch (IOException) {
} catch (UnauthorizedAccessException) {
}
if (Directory.Exists(factory.DatabasePath))
{
var analysisLog = Path.Combine(factory.DatabasePath, "AnalysisLog.txt");
for (int retries = 10; retries > 0; --retries)
{
try {
File.AppendAllText(analysisLog, string.Format(
"Exception while loading extension module {0}{1}{2}{1}",
reference.Name,
Environment.NewLine,
ex
));
break;
} catch (Exception ex2) {
if (ex2.IsCriticalException())
{
throw;
}
}
}
}
}
}
}
var evt = ModuleNamesChanged;
if (evt != null)
{
evt(this, EventArgs.Empty);
}
}
private void EnsureSearchPathDB() {
if (_searchPathDb == null) {
_searchPathDb = new PythonTypeDatabase(_factory, innerDatabase: _typeDb);
}
}
public CPythonType(IMemberContainer parent, ITypeDatabaseReader typeDb, string typeName, Dictionary <string, object> typeTable, BuiltinTypeId typeId)
{
Debug.Assert(parent is CPythonType || parent is CPythonModule);
Debug.Assert(!typeId.IsVirtualId());
_typeName = typeName;
_typeId = typeId;
_module = GetDeclaringModule(parent);
object value;
if (typeTable.TryGetValue("is_hidden", out value))
{
_includeInModule = !Convert.ToBoolean(value);
}
else
{
_includeInModule = true;
}
if (typeTable.TryGetValue("doc", out value))
{
_doc = value as string;
}
if (typeTable.TryGetValue("builtin", out value))
{
_isBuiltin = Convert.ToBoolean(value);
}
else
{
_isBuiltin = true;
}
if (typeTable.TryGetValue("bases", out value))
{
var basesList = (List <object>)value;
if (basesList != null)
{
_bases = new List <IPythonType>();
foreach (var baseType in basesList)
{
typeDb.LookupType(baseType, StoreBase);
}
}
}
if (typeTable.TryGetValue("mro", out value))
{
var mroList = (List <object>)value;
if (mroList != null && mroList.Count >= 1)
{
_mro = new IPythonType[mroList.Count];
// Many scraped types have invalid MRO entries because they
// report their own module/name incorrectly. Since the first
// item in the MRO is always self, we set it now. If the MRO
// has a resolvable entry it will replace this one.
_mro[0] = this;
for (int i = 0; i < mroList.Count; ++i)
{
var capturedI = i;
typeDb.LookupType(mroList[i], t => _mro[capturedI] = t);
}
}
}
if (typeTable.TryGetValue("members", out value))
{
var membersTable = (Dictionary <string, object>)value;
if (membersTable != null)
{
LoadMembers(typeDb, membersTable);
}
}
_hasLocation = PythonTypeDatabase.TryGetLocation(typeTable, ref _line, ref _column);
}
public async Task <List <string> > EnumerateAllModules(bool refresh = false)
{
AbortOnInvalidConfiguration();
if (_modules == null || refresh)
{
var stdLibPaths = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
stdLibPaths.Add(_factory.Configuration.LibraryPath);
stdLibPaths.Add(Path.Combine(_factory.Configuration.PrefixPath, "DLLs"));
stdLibPaths.Add(Path.GetDirectoryName(_factory.Configuration.InterpreterPath));
var results = await Task.Run(() => {
List <PythonLibraryPath> paths;
if (_factory.AssumeSimpleLibraryLayout)
{
paths = PythonTypeDatabase.GetDefaultDatabaseSearchPaths(_factory.Configuration.LibraryPath);
}
else
{
paths = PythonTypeDatabase.GetCachedDatabaseSearchPaths(_factory.DatabasePath);
if (paths == null)
{
paths = PythonTypeDatabase.GetUncachedDatabaseSearchPathsAsync(
_factory.Configuration.InterpreterPath
).WaitAndUnwrapExceptions();
try {
PythonTypeDatabase.WriteDatabaseSearchPaths(_factory.DatabasePath, paths);
} catch (Exception ex) {
if (ex.IsCriticalException())
{
throw;
}
}
}
}
var groups = PythonTypeDatabase.GetDatabaseExpectedModules(
_factory.Configuration.Version,
paths
).ToList();
var stdLibModules = groups[0].Select(mp => mp.ModuleName).ToList();
var modules = groups.SelectMany().Select(mp => mp.ModuleName).ToList();
stdLibModules.Sort();
modules.Sort();
for (int i = stdLibModules.Count - 1; i > 0; --i)
{
if (stdLibModules[i - 1] == stdLibModules[i])
{
stdLibModules.RemoveAt(i);
}
}
for (int i = modules.Count - 1; i > 0; --i)
{
if (modules[i - 1] == modules[i])
{
modules.RemoveAt(i);
}
}
return(Tuple.Create(modules, stdLibModules));
});
_modules = results.Item1;
_stdLibModules = results.Item2;
}
return(_modules);
}
public AnalysisOnlyInterpreterFactory(Version version, PythonTypeDatabase database, string description = null)
: base(GetConfiguration(version, database.DatabaseDirectory), CreationOptions)
{
_actualDatabase = database;
}
public void VersionedSharedDatabase()
{
var twoFive = PythonTypeDatabase.CreateDefaultTypeDatabase(new Version(2, 5));
var twoSix = PythonTypeDatabase.CreateDefaultTypeDatabase(new Version(2, 6));
var twoSeven = PythonTypeDatabase.CreateDefaultTypeDatabase(new Version(2, 7));
var threeOh = PythonTypeDatabase.CreateDefaultTypeDatabase(new Version(3, 0));
var threeOne = PythonTypeDatabase.CreateDefaultTypeDatabase(new Version(3, 1));
var threeTwo = PythonTypeDatabase.CreateDefaultTypeDatabase(new Version(3, 2));
// new in 2.6
Assert.AreEqual(null, twoFive.BuiltinModule.GetAnyMember("bytearray"));
foreach (var version in new[] { twoSix, twoSeven, threeOh, threeOne, threeTwo })
{
Assert.AreNotEqual(version, version.BuiltinModule.GetAnyMember("bytearray"));
}
// new in 2.7
Assert.AreEqual(null, twoSix.BuiltinModule.GetAnyMember("memoryview"));
foreach (var version in new[] { twoSeven, threeOh, threeOne, threeTwo })
{
Assert.AreNotEqual(version, version.BuiltinModule.GetAnyMember("memoryview"));
}
// not in 3.0
foreach (var version in new[] { twoFive, twoSix, twoSeven })
{
Assert.AreNotEqual(null, version.BuiltinModule.GetAnyMember("StandardError"));
}
foreach (var version in new[] { threeOh, threeOne, threeTwo })
{
Assert.AreEqual(null, version.BuiltinModule.GetAnyMember("StandardError"));
}
// new in 3.0
foreach (var version in new[] { twoFive, twoSix, twoSeven })
{
Assert.AreEqual(null, version.BuiltinModule.GetAnyMember("exec"));
Assert.AreEqual(null, version.BuiltinModule.GetAnyMember("print"));
}
foreach (var version in new[] { threeOh, threeOne, threeTwo })
{
Assert.AreNotEqual(null, version.BuiltinModule.GetAnyMember("exec"));
Assert.AreNotEqual(null, version.BuiltinModule.GetAnyMember("print"));
}
// new in 3.1
foreach (var version in new[] { twoFive, twoSix, twoSeven, threeOh })
{
Assert.AreEqual(null, version.GetModule("sys").GetMember(null, "int_info"));
}
foreach (var version in new[] { threeOne, threeTwo })
{
Assert.AreNotEqual(null, version.GetModule("sys").GetMember(null, "int_info"));
}
// new in 3.2
foreach (var version in new[] { twoFive, twoSix, twoSeven, threeOh, threeOne })
{
Assert.AreEqual(null, version.GetModule("sys").GetMember(null, "setswitchinterval"));
}
foreach (var version in new[] { threeTwo })
{
Assert.AreNotEqual(null, version.GetModule("sys").GetMember(null, "setswitchinterval"));
}
}
public void PydInPackage()
{
PythonPaths.Python27.AssertInstalled();
var outputPath = TestData.GetTempPath();
Console.WriteLine("Writing to: " + outputPath);
// run the analyzer
using (var output = ProcessOutput.RunHiddenAndCapture("Microsoft.PythonTools.Analyzer.exe",
"/python", PythonPaths.Python27.InterpreterPath,
"/lib", TestData.GetPath(@"TestData\PydStdLib"),
"/version", "2.7",
"/outdir", outputPath,
"/indir", CompletionDB,
"/unittest",
"/log", "AnalysisLog.txt")) {
output.Wait();
Console.WriteLine("* Stdout *");
foreach (var line in output.StandardOutputLines)
{
Console.WriteLine(line);
}
Console.WriteLine("* Stderr *");
foreach (var line in output.StandardErrorLines)
{
Console.WriteLine(line);
}
Assert.AreEqual(0, output.ExitCode);
}
var fact = PythonInterpreterFactoryWithDatabase.CreateFromDatabase(new Version(2, 7));
var paths = new List <string> {
outputPath
};
paths.AddRange(Directory.EnumerateDirectories(outputPath));
var typeDb = new PythonTypeDatabase(fact, paths);
var module = typeDb.GetModule("Package.winsound");
Assert.IsNotNull(module, "Package.winsound was not analyzed");
var package = typeDb.GetModule("Package");
Assert.IsNotNull(package, "Could not import Package");
var member = package.GetMember(null, "winsound");
Assert.IsNotNull(member, "Could not get member Package.winsound");
Assert.AreSame(module, member);
module = typeDb.GetModule("Package._testcapi");
Assert.IsNotNull(module, "Package._testcapi was not analyzed");
package = typeDb.GetModule("Package");
Assert.IsNotNull(package, "Could not import Package");
member = package.GetMember(null, "_testcapi");
Assert.IsNotNull(member, "Could not get member Package._testcapi");
Assert.IsNotInstanceOfType(member, typeof(CPythonMultipleMembers));
Assert.AreSame(module, member);
module = typeDb.GetModule("Package.select");
Assert.IsNotNull(module, "Package.select was not analyzed");
package = typeDb.GetModule("Package");
Assert.IsNotNull(package, "Could not import Package");
member = package.GetMember(null, "select");
Assert.IsNotNull(member, "Could not get member Package.select");
Assert.IsInstanceOfType(member, typeof(CPythonMultipleMembers));
var mm = (CPythonMultipleMembers)member;
AssertUtil.ContainsExactly(mm.Members.Select(m => m.MemberType),
PythonMemberType.Module,
PythonMemberType.Constant,
PythonMemberType.Class
);
Assert.IsNotNull(mm.Members.Contains(module));
try {
// Only clean up if the test passed
Directory.Delete(outputPath, true);
} catch { }
}
public void Dispose() {
_searchPathDb = null;
_zipPackageCache = null;
_typeDb = null;
var factory = _factory;
_factory = null;
if (factory != null) {
factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
}
}
public CPythonInterpreter(PythonInterpreterFactoryWithDatabase factory) {
_langVersion = factory.Configuration.Version;
_factory = factory;
_typeDb = _factory.GetCurrentDatabase();
_factory.NewDatabaseAvailable += OnNewDatabaseAvailable;
}
public AnalysisOnlyInterpreterFactory(Version version, PythonTypeDatabase database, string description = null)
: this(version, description) {
_actualDatabase = database;
}
public AnalysisOnlyInterpreterFactory(Version version, PythonTypeDatabase database, string description = null)
: this(version, description)
{
_actualDatabase = database;
}
