private bool CanUpdateAnalysis(
PythonAnalyzerEntry entry,
IDependencyChainNode <PythonAnalyzerEntry> node,
int version,
out IPythonModule module,
out PythonAst ast,
out IDocumentAnalysis currentAnalysis)
{
if (!entry.CanUpdateAnalysis(version, out module, out ast, out currentAnalysis))
{
if (IsAnalyzedLibraryInLoop(node, currentAnalysis))
{
// Library analysis exists, don't analyze again
return(false);
}
if (ast == null)
{
if (currentAnalysis == null)
{
// Entry doesn't have ast yet. There should be at least one more session.
Cancel();
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled (no AST yet).");
return(false);
}
//Debug.Fail($"Library module {module.Name} of type {module.ModuleType} has been analyzed already!");
return(false);
}
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled. Version: {version}, current: {module.Analysis.Version}.");
return(false);
}
return(true);
}
private IDocumentAnalysis CreateAnalysis(IDependencyChainNode <PythonAnalyzerEntry> node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled)
{
var canHaveLibraryAnalysis = false;
// Don't try to drop builtins; it causes issues elsewhere.
// We probably want the builtin module's AST and other info for evaluation.
switch (document.ModuleType)
{
case ModuleType.Library:
case ModuleType.Stub:
case ModuleType.Compiled:
canHaveLibraryAnalysis = true;
break;
}
var createLibraryAnalysis = !isCanceled &&
node != null &&
!node.HasMissingDependencies &&
canHaveLibraryAnalysis &&
!document.IsOpen &&
node.HasOnlyWalkedDependencies &&
node.IsValidVersion;
if (!createLibraryAnalysis)
{
return(new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames));
}
ast.Reduce(x => x is ImportStatement || x is FromImportStatement);
document.SetAst(ast);
var eval = new ExpressionEval(walker.Eval.Services, document, ast);
return(new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames));
}
/// <summary>
/// Performs analysis of the document. Returns document global scope
/// with declared variables and inner scopes. Does not analyze chain
/// of dependencies, it is intended for the single file analysis.
/// </summary>
private void Analyze(IDependencyChainNode <PythonAnalyzerEntry> node, AsyncCountdownEvent ace, Stopwatch stopWatch)
{
try {
var entry = node.Value;
if (!CanUpdateAnalysis(entry, node, _walker.Version, out var module, out var ast, out var currentAnalysis))
{
return;
}
var startTime = stopWatch.Elapsed;
AnalyzeEntry(node, entry, module, ast, _walker.Version);
LogCompleted(node, module, stopWatch, startTime);
} catch (OperationCanceledException oce) {
node.Value.TryCancel(oce, _walker.Version);
LogCanceled(node.Value.Module);
} catch (Exception exception) {
node.Value.TrySetException(exception, _walker.Version);
node.MarkWalked();
LogException(node.Value.Module, exception);
} finally {
node.MoveNext();
lock (_syncObj) {
if (!_isCanceled)
{
_progress.ReportRemaining(_walker.Remaining);
}
_runningTasks--;
ace?.Signal();
}
}
}
/// <summary>
/// Performs analysis of the document. Returns document global scope
/// with declared variables and inner scopes. Does not analyze chain
/// of dependencies, it is intended for the single file analysis.
/// </summary>
private async Task <IDocumentAnalysis> AnalyzeAsync(IDependencyChainNode node, CancellationToken cancellationToken)
{
var _startTime = DateTime.Now;
_log?.Log(TraceEventType.Verbose, $"Analysis begins: {node.Document.Name}({node.Document.ModuleType})");
// Store current expected version so we can see if it still
// the same at the time the analysis completes.
var analysisVersion = node.Analyzable.ExpectedAnalysisVersion;
// Make sure the file is parsed ans the AST is up to date.
var ast = await node.Document.GetAstAsync(cancellationToken);
_log?.Log(TraceEventType.Verbose, $"Parse of {node.Document.Name}({node.Document.ModuleType}) complete in {(DateTime.Now - _startTime).TotalMilliseconds} ms.");
// Now run the analysis.
var walker = new ModuleWalker(_services, node.Document, ast);
await ast.WalkAsync(walker, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
// Note that we do not set the new analysis here and rather let
// Python analyzer to call NotifyAnalysisComplete.
await walker.CompleteAsync(cancellationToken);
_log?.Log(TraceEventType.Verbose, $"Analysis of {node.Document.Name}({node.Document.ModuleType}) complete in {(DateTime.Now - _startTime).TotalMilliseconds} ms.");
return(new DocumentAnalysis(node.Document, analysisVersion, walker.GlobalScope, walker.Ast));
}
private IDocumentAnalysis TryRestoreCachedAnalysis(IDependencyChainNode <PythonAnalyzerEntry> node, IPythonModule module)
{
var moduleType = module.ModuleType;
if (moduleType.CanBeCached() && _moduleDatabaseService?.ModuleExistsInStorage(module.Name, module.FilePath) == true)
{
if (_moduleDatabaseService.TryRestoreGlobalScope(module, out var gs))
{
if (_log != null)
{
_log.Log(TraceEventType.Verbose, "Restored from database: ", module.Name);
}
var analysis = new DocumentAnalysis((IDocument)module, 1, gs, new ExpressionEval(_services, module, module.GetAst()), Array.Empty <string>());
gs.ReconstructVariables();
MarkNodeWalked(node);
return(analysis);
}
else
{
if (_log != null)
{
_log.Log(TraceEventType.Verbose, "Restore from database failed for module ", module.Name);
}
}
}
return(null);
}
private void NotifyAnalysisPending(IDependencyChainNode node)
{
node.Analyzable.NotifyAnalysisPending();
foreach (var c in node.Children)
{
NotifyAnalysisPending(c);
}
}
/// <summary>
/// Performs analysis of the document. Returns document global scope
/// with declared variables and inner scopes. Does not analyze chain
/// of dependencies, it is intended for the single file analysis.
/// </summary>
private void Analyze(IDependencyChainNode <PythonAnalyzerEntry> node, AsyncCountdownEvent ace, Stopwatch stopWatch)
{
try {
var entry = node.Value;
if (!entry.CanUpdateAnalysis(_walker.Version, out var module, out var ast, out var currentAnalysis))
{
if (IsAnalyzedLibraryInLoop(node, currentAnalysis))
{
return;
}
else if (ast == default)
{
if (currentAnalysis == default)
{
// Entry doesn't have ast yet. There should be at least one more session.
Cancel();
}
else
{
Debug.Fail($"Library module {module.Name} of type {module.ModuleType} has been analyzed already!");
}
}
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
return;
}
var startTime = stopWatch.Elapsed;
AnalyzeEntry(node, entry, module, ast, _walker.Version);
LogCompleted(node, module, stopWatch, startTime);
} catch (OperationCanceledException oce) {
node.Value.TryCancel(oce, _walker.Version);
LogCanceled(node.Value.Module);
} catch (Exception exception) {
node.Value.TrySetException(exception, _walker.Version);
node.MarkWalked();
LogException(node.Value.Module, exception);
} finally {
node.MoveNext();
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled)
{
_progress.ReportRemaining(_walker.Remaining);
}
Interlocked.Decrement(ref _runningTasks);
ace?.Signal();
}
}
private static void NotifyAnalysisComplete(IDependencyChainNode node, IDocumentAnalysis analysis)
{
if (!node.Analyzable.NotifyAnalysisComplete(analysis))
{
// If snapshot does not match, there is no reason to continue analysis along the chain
// since subsequent change that incremented the expected version will start
// another analysis run.
throw new OperationCanceledException();
}
}
private void LogCompleted(IDependencyChainNode <PythonAnalyzerEntry> node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime)
{
if (_log != null)
{
var completed = node != null && module.Analysis is LibraryAnalysis ? "completed for library" : "completed ";
var message = node != null
? $"Analysis of {module.Name}({module.ModuleType}) on depth {node.VertexDepth} {completed} in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."
: $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms.";
_log.Log(TraceEventType.Verbose, message);
}
}
private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch)
{
var loopAnalysis = false;
try {
switch (node)
{
case IDependencyChainSingleNode <PythonAnalyzerEntry> single:
try {
Analyze(single, stopWatch);
} catch (OperationCanceledException oce) {
single.Value.TryCancel(oce, _walker.Version);
LogCanceled(single.Value.Module);
} catch (Exception exception) {
single.Value.TrySetException(exception, _walker.Version);
node.MarkWalked();
LogException(single.Value, exception);
}
break;
case IDependencyChainLoopNode <PythonAnalyzerEntry> loop:
try {
loopAnalysis = true;
AnalyzeLoop(loop, stopWatch);
} catch (OperationCanceledException) {
//loop.Value.TryCancel(oce, _walker.Version);
//LogCanceled(single.Value.Module);
} catch (Exception exception) {
//loop.Value.TrySetException(exception, _walker.Version);
node.MarkWalked();
LogException(loop, exception);
}
break;
}
} finally {
node.MoveNext();
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled || loopAnalysis)
{
_progress.ReportRemaining(_walker.Remaining);
}
Interlocked.Decrement(ref _runningTasks);
ace?.Signal();
}
}
private bool MarkNodeWalked(IDependencyChainNode <PythonAnalyzerEntry> node)
{
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled)
{
node?.MarkWalked();
}
return(isCanceled);
}
private async Task AnalyzeChainAsync(IDependencyChainNode node, CancellationToken cancellationToken)
{
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(_globalCts.Token, cancellationToken)) {
var analysis = await AnalyzeAsync(node, cts.Token).ConfigureAwait(false);
NotifyAnalysisComplete(node, analysis);
cts.Token.ThrowIfCancellationRequested();
foreach (var c in node.Children)
{
await AnalyzeChainAsync(c, cts.Token).ConfigureAwait(false);
}
}
}
/// <summary>
/// Performs analysis of the document. Returns document global scope
/// with declared variables and inner scopes. Does not analyze chain
/// of dependencies, it is intended for the single file analysis.
/// </summary>
private void Analyze(IDependencyChainNode <PythonAnalyzerEntry> node, AsyncCountdownEvent ace, Stopwatch stopWatch)
{
IPythonModule module;
try {
ace?.AddOne();
var entry = node.Value;
if (!entry.IsValidVersion(_walker.Version, out module, out var ast))
{
if (ast == null)
{
// Entry doesn't have ast yet. There should be at least one more session.
Cancel();
}
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
node.Skip();
return;
}
var startTime = stopWatch.Elapsed;
AnalyzeEntry(entry, module, ast, _walker.Version);
node.Commit();
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms.");
} catch (OperationCanceledException oce) {
node.Value.TryCancel(oce, _walker.Version);
node.Skip();
module = node.Value.Module;
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
} catch (Exception exception) {
module = node.Value.Module;
node.Value.TrySetException(exception, _walker.Version);
node.Commit();
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. Exception message: {exception.Message}.");
} finally {
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled)
{
_progress.ReportRemaining(_walker.Remaining);
}
Interlocked.Decrement(ref _runningTasks);
ace?.Signal();
}
}
private IDocumentAnalysis DoAnalyzeEntry(IDependencyChainNode <PythonAnalyzerEntry> node, IPythonModule module, PythonAst ast, int version)
{
var analysis = TryRestoreCachedAnalysis(node, module);
if (analysis != null)
{
return(analysis);
}
var walker = new ModuleWalker(_services, module, ast, _analyzerCancellationToken);
ast.Walk(walker);
walker.Complete();
return(CreateAnalysis(node, (IDocument)module, ast, version, walker));
}
private IDocumentAnalysis CreateAnalysis(IDependencyChainNode <PythonAnalyzerEntry> node, IDocument document, PythonAst ast, int version, ModuleWalker walker)
{
var canHaveLibraryAnalysis = false;
// Don't try to drop builtins; it causes issues elsewhere.
// We probably want the builtin module's AST and other info for evaluation.
switch (document.ModuleType)
{
case ModuleType.Library:
case ModuleType.Compiled:
case ModuleType.CompiledBuiltin:
canHaveLibraryAnalysis = true;
break;
}
var isCanceled = MarkNodeWalked(node);
var createLibraryAnalysis = !isCanceled &&
node != null &&
!node.HasMissingDependencies &&
canHaveLibraryAnalysis &&
!document.IsOpen &&
node.HasOnlyWalkedDependencies &&
node.IsValidVersion;
var optionsProvider = _services.GetService <IAnalysisOptionsProvider>();
if (optionsProvider?.Options.KeepLibraryAst == true)
{
createLibraryAnalysis = false;
}
if (!createLibraryAnalysis)
{
return(new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames));
}
ast.Reduce(x => x is ImportStatement || x is FromImportStatement);
document.SetAst(ast);
var eval = new ExpressionEval(walker.Eval.Services, document, ast);
var analysis = new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames);
var dbs = _services.GetService <IModuleDatabaseService>();
dbs?.StoreModuleAnalysisAsync(analysis, CancellationToken.None).DoNotWait();
return(analysis);
}
/// <summary>
/// Performs analysis of the document. Returns document global scope
/// with declared variables and inner scopes. Does not analyze chain
/// of dependencies, it is intended for the single file analysis.
/// </summary>
private void Analyze(IDependencyChainNode <PythonAnalyzerEntry> node, AsyncCountdownEvent ace, Stopwatch stopWatch)
{
try {
ace?.AddOne();
var entry = node.Value;
if (!entry.IsValidVersion(_walker.Version, out var module, out var ast))
{
if (ast == null)
{
// Entry doesn't have ast yet. There should be at least one more session.
Cancel();
}
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
node.Skip();
return;
}
var startTime = stopWatch.Elapsed;
AnalyzeEntry(entry, module, _walker.Version, node.IsComplete);
node.Commit();
ActivityTracker.OnModuleAnalysisComplete(node.Value.Module.FilePath);
LogCompleted(module, stopWatch, startTime);
} catch (OperationCanceledException oce) {
node.Value.TryCancel(oce, _walker.Version);
node.Skip();
LogCanceled(node.Value.Module);
} catch (Exception exception) {
node.Value.TrySetException(exception, _walker.Version);
node.Commit();
LogException(node.Value.Module, exception);
} finally {
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled)
{
_progress.ReportRemaining(_walker.Remaining);
}
Interlocked.Decrement(ref _runningTasks);
ace?.Signal();
}
}
private IDocumentAnalysis DoAnalyzeEntry(IDependencyChainNode <PythonAnalyzerEntry> node, IPythonModule module, PythonAst ast, int version)
{
var analysis = TryRestoreCachedAnalysis(node, module);
if (analysis != null)
{
return(analysis);
}
if (module is IAnalyzable analyzable)
{
var walker = analyzable.Analyze(ast);
return(CreateAnalysis(node, (IDocument)module, ast, version, walker));
}
return(new EmptyAnalysis(_services, (IDocument)module));
}
private void Analyze(IDependencyChainNode node, Stopwatch stopWatch)
{
var loopAnalysis = false;
try {
switch (node)
{
case IDependencyChainSingleNode <PythonAnalyzerEntry> single:
try {
Analyze(single, stopWatch);
} catch (OperationCanceledException oce) {
single.Value.TryCancel(oce, _walker.Version);
LogCanceled(single.Value.Module);
} catch (Exception exception) {
single.Value.TrySetException(exception, _walker.Version);
node.MarkWalked();
LogException(single.Value, exception);
}
break;
case IDependencyChainLoopNode <PythonAnalyzerEntry> loop:
try {
loopAnalysis = true;
AnalyzeLoop(loop, stopWatch);
} catch (OperationCanceledException) {
} catch (Exception exception) {
node.MarkWalked();
LogException(loop, exception);
}
break;
}
} finally {
lock (_syncObj) {
node.MoveNext();
if (!_isCanceled || loopAnalysis)
{
_progress.ReportRemaining(_walker.Remaining);
}
_ace.Signal();
}
}
}
private IDocumentAnalysis CreateAnalysis(IDependencyChainNode <PythonAnalyzerEntry> node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled)
{
var createLibraryAnalysis = !isCanceled &&
node != null &&
!node.HasMissingDependencies &&
document.ModuleType == ModuleType.Library &&
!document.IsOpen &&
node.HasOnlyWalkedDependencies &&
node.IsValidVersion;
if (!createLibraryAnalysis)
{
return(new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames));
}
ast.Reduce(x => x is ImportStatement || x is FromImportStatement);
document.SetAst(ast);
var eval = new ExpressionEval(walker.Eval.Services, document, ast);
return(new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames));
}
private void AnalyzeEntry(IDependencyChainNode <PythonAnalyzerEntry> node, PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version)
{
// Now run the analysis.
var analyzable = module as IAnalyzable;
analyzable?.NotifyAnalysisBegins();
var walker = new ModuleWalker(_services, module, ast);
ast.Walk(walker);
_analyzerCancellationToken.ThrowIfCancellationRequested();
walker.Complete();
_analyzerCancellationToken.ThrowIfCancellationRequested();
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled)
{
node?.MarkWalked();
}
var analysis = CreateAnalysis(node, (IDocument)module, ast, version, walker, isCanceled);
analyzable?.NotifyAnalysisComplete(analysis);
entry.TrySetAnalysis(module.Analysis, version);
if (module.ModuleType == ModuleType.User)
{
var linterDiagnostics = _analyzer.LintModule(module);
_diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter);
}
}
private void AnalyzeEntry(IDependencyChainNode <PythonAnalyzerEntry> node, PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version)
{
// Now run the analysis.
var analyzable = module as IAnalyzable;
analyzable?.NotifyAnalysisBegins();
Debug.Assert(ast != null);
var analysis = DoAnalyzeEntry(node, module, ast, version);
_analyzerCancellationToken.ThrowIfCancellationRequested();
if (analysis != null)
{
analyzable?.NotifyAnalysisComplete(analysis);
entry.TrySetAnalysis(analysis, version);
if (module.ModuleType == ModuleType.User)
{
var linterDiagnostics = _analyzer.LintModule(module);
_diagnosticsService?.Replace(entry.Module.Uri, linterDiagnostics, DiagnosticSource.Linter);
}
}
}
public static DependencyChainNodeAssertions Should(this IDependencyChainNode node) => new DependencyChainNodeAssertions(node);
private Task StartAnalysis(IDependencyChainNode <PythonAnalyzerEntry> node, AsyncCountdownEvent ace, Stopwatch stopWatch, CancellationToken cancellationToken) => Task.Run(() => Analyze(node, ace, stopWatch, cancellationToken), cancellationToken);
private Task StartAnalysis(IDependencyChainNode <PythonAnalyzerEntry> node, AsyncCountdownEvent ace, Stopwatch stopWatch) => Task.Run(() => Analyze(node, ace, stopWatch));
private bool IsAnalyzedLibraryInLoop(IDependencyChainNode <PythonAnalyzerEntry> node) => !node.HasMissingDependencies && node.Value.IsAnalyzedLibrary(_walker.Version) && node.IsWalkedWithDependencies && node.IsValidVersion;
private Task StartAnalysis(IDependencyChainWalker <AnalysisModuleKey, PythonAnalyzerEntry> walker, IDependencyChainNode <PythonAnalyzerEntry> node, Stopwatch stopWatch, CancellationToken cancellationToken) => Task.Run(() => Analyze(walker, node, stopWatch, cancellationToken), cancellationToken);
/// <summary>
/// Performs analysis of the document. Returns document global scope
/// with declared variables and inner scopes. Does not analyze chain
/// of dependencies, it is intended for the single file analysis.
/// </summary>
private void Analyze(IDependencyChainWalker <AnalysisModuleKey, PythonAnalyzerEntry> walker, IDependencyChainNode <PythonAnalyzerEntry> node, Stopwatch stopWatch, CancellationToken cancellationToken)
{
IPythonModule module;
try {
var entry = node.Value;
if (!entry.IsValidVersion(walker.Version, out module, out var ast))
{
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
node.Skip();
return;
}
var startTime = stopWatch.Elapsed;
AnalyzeEntry(entry, module, ast, walker.Version, cancellationToken);
node.Commit();
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms.");
} catch (OperationCanceledException oce) {
node.Value.TryCancel(oce, walker.Version);
node.Skip();
module = node.Value.Module;
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
} catch (Exception exception) {
module = node.Value.Module;
node.Value.TrySetException(exception, walker.Version);
node.Commit();
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed.");
} finally {
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled)
{
_progress.ReportRemaining(walker.Remaining);
}
Interlocked.Decrement(ref _runningTasks);
}
}
private Task StartAnalysis(IDependencyChainNode node, Stopwatch stopWatch) => Task.Run(() => Analyze(node, stopWatch));
private bool IsAnalyzedLibraryInLoop(IDependencyChainNode <PythonAnalyzerEntry> node, IDocumentAnalysis currentAnalysis) => !node.HasMissingDependencies && currentAnalysis is LibraryAnalysis && node.IsWalkedWithDependencies && node.IsValidVersion;
private void RunAnalysis(IDependencyChainNode <PythonAnalyzerEntry> node, Stopwatch stopWatch) => ExecutionContext.Run(ExecutionContext.Capture(), s => Analyze(node, null, stopWatch), null);
