private async Task <int> AnalyzeAffectedEntriesAsync(Stopwatch stopWatch)
{
IDependencyChainNode <PythonAnalyzerEntry> node;
var remaining = 0;
var ace = new AsyncCountdownEvent(0);
bool isCanceled;
while ((node = await _walker.GetNextAsync(_analyzerCancellationToken)) != null)
{
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (isCanceled && !node.Value.NotAnalyzed)
{
remaining++;
node.MoveNext();
continue;
}
ActivityTracker.OnEnqueueModule(node.Value.Module.FilePath);
if (Interlocked.Increment(ref _runningTasks) >= _maxTaskRunning || _walker.Remaining == 1)
{
RunAnalysis(node, stopWatch);
}
else
{
ace.AddOne();
StartAnalysis(node, ace, stopWatch).DoNotWait();
}
}
await ace.WaitAsync(_analyzerCancellationToken);
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (_walker.MissingKeys.Count == 0 || _walker.MissingKeys.All(k => k.IsTypeshed))
{
Interlocked.Exchange(ref _runningTasks, 0);
if (!isCanceled)
{
_analysisCompleteEvent.Set();
}
}
else if (!isCanceled && _log != null && _log.LogLevel >= TraceEventType.Verbose)
{
_log?.Log(TraceEventType.Verbose, $"Missing keys: {string.Join(", ", _walker.MissingKeys)}");
}
return(remaining);
}
private async Task StartAsync()
{
_progress.ReportRemaining(_walker.Remaining);
lock (_syncObj) {
var notAnalyzed = _walker.AffectedValues.Count(e => e.NotAnalyzed);
if (_isCanceled && notAnalyzed < _maxTaskRunning)
{
_state = State.Completed;
return;
}
}
var stopWatch = Stopwatch.StartNew();
foreach (var affectedEntry in _walker.AffectedValues)
{
affectedEntry.Invalidate(Version);
}
var originalRemaining = _walker.Remaining;
var remaining = originalRemaining;
try {
_log?.Log(TraceEventType.Verbose, $"Analysis version {Version} of {originalRemaining} entries has started.");
remaining = await AnalyzeAffectedEntriesAsync(stopWatch);
} finally {
stopWatch.Stop();
bool isCanceled;
bool isFinal;
lock (_syncObj) {
isCanceled = _isCanceled;
_state = State.Completed;
isFinal = _walker.MissingKeys.Count == 0 && !isCanceled && remaining == 0;
_walker = null;
}
if (!isCanceled)
{
_progress.ReportRemaining(remaining);
if (isFinal)
{
var(modulesCount, totalMilliseconds) = ActivityTracker.EndTracking();
(_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds);
_log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms.");
}
}
}
var elapsed = stopWatch.Elapsed.TotalMilliseconds;
LogResults(_log, elapsed, originalRemaining, remaining, Version);
ForceGCIfNeeded(originalRemaining, remaining);
}
private async Task StartAsync()
{
_progress.ReportRemaining(_walker.Remaining);
lock (_syncObj) {
var notAnalyzed = _walker.AffectedValues.Count(e => e.NotAnalyzed);
if (_isCanceled && notAnalyzed < _maxTaskRunning)
{
_state = State.Completed;
return;
}
}
var stopWatch = Stopwatch.StartNew();
var originalRemaining = _walker.Remaining;
var remaining = originalRemaining;
try {
_log?.Log(TraceEventType.Verbose, $"Analysis version {Version} of {originalRemaining} entries has started.");
remaining = await AnalyzeAffectedEntriesAsync(stopWatch);
} finally {
stopWatch.Stop();
bool isCanceled;
bool isFinal;
lock (_syncObj) {
isCanceled = _isCanceled;
_state = State.Completed;
isFinal = _walker.MissingKeys.Count == 0 && !isCanceled && remaining == 0;
_walker = null;
}
if (!isCanceled)
{
_progress.ReportRemaining(remaining);
if (isFinal)
{
var(modulesCount, totalMilliseconds) = ActivityTracker.EndTracking();
totalMilliseconds = Math.Round(totalMilliseconds, 2);
(_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds);
_log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms.");
//#if DEBUG
// var notReady = _analyzer.LoadedModules.Where(m => (m.ModuleType == ModuleType.Library || m.ModuleType == ModuleType.Stub) && m.Analysis is EmptyAnalysis).ToArray();
// Debug.Assert(notReady.Length == 0);
//#endif
}
}
}
var elapsed = stopWatch.Elapsed.TotalMilliseconds;
LogResults(_log, elapsed, originalRemaining, remaining, Version);
ForceGCIfNeeded(_log, originalRemaining, remaining, _forceGC);
}
private async Task StartAsync()
{
_progress.ReportRemaining(_walker.Remaining);
lock (_syncObj) {
var notAnalyzed = _walker.AffectedValues.Count(e => e.NotAnalyzed);
if (_isCanceled && notAnalyzed < _maxTaskRunning)
{
_state = State.Completed;
return;
}
}
var stopWatch = Stopwatch.StartNew();
var originalRemaining = _walker.Remaining;
var remaining = originalRemaining;
try {
_log?.Log(TraceEventType.Verbose, $"Analysis version {Version} of {originalRemaining} entries has started.");
remaining = await AnalyzeAffectedEntriesAsync(stopWatch);
Debug.Assert(_ace.Count == 0);
} finally {
stopWatch.Stop();
var isFinal = false;
lock (_syncObj) {
if (!_isCanceled)
{
_progress.ReportRemaining(remaining);
}
_state = State.Completed;
isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0;
_walker = null;
}
if (isFinal)
{
var(modulesCount, totalMilliseconds) = ActivityTracker.EndTracking();
totalMilliseconds = Math.Round(totalMilliseconds, 2);
if (await _analyzer.RaiseAnalysisCompleteAsync(modulesCount, totalMilliseconds))
{
_log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms.");
}
}
}
var elapsed = stopWatch.Elapsed.TotalMilliseconds;
LogResults(_log, elapsed, originalRemaining, remaining, Version);
ForceGCIfNeeded(_log, originalRemaining, remaining, _forceGC);
}
/// <summary>
/// Performs analysis of the document. Returns document global scope
/// with declared variables and inner scopes. Does not analyze chain
/// of dependencies, it is intended for the single file analysis.
/// </summary>
private void Analyze(IDependencyChainSingleNode <PythonAnalyzerEntry> node, Stopwatch stopWatch)
{
ActivityTracker.OnEnqueueModule(node.Value.Module.FilePath);
var entry = node.Value;
if (!CanUpdateAnalysis(entry, _walker.Version, out var module, out var ast))
{
return;
}
var startTime = stopWatch.Elapsed;
AnalyzeEntry(node, entry, module, ast, _walker.Version);
LogCompleted(node, module, stopWatch, startTime);
}
/// <summary>
/// Performs analysis of the document. Returns document global scope
/// with declared variables and inner scopes. Does not analyze chain
/// of dependencies, it is intended for the single file analysis.
/// </summary>
private void Analyze(IDependencyChainNode <PythonAnalyzerEntry> node, AsyncCountdownEvent ace, Stopwatch stopWatch)
{
try {
ace?.AddOne();
var entry = node.Value;
if (!entry.IsValidVersion(_walker.Version, out var module, out var ast))
{
if (ast == null)
{
// Entry doesn't have ast yet. There should be at least one more session.
Cancel();
}
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
node.Skip();
return;
}
var startTime = stopWatch.Elapsed;
AnalyzeEntry(entry, module, _walker.Version, node.IsComplete);
node.Commit();
ActivityTracker.OnModuleAnalysisComplete(node.Value.Module.FilePath);
LogCompleted(module, stopWatch, startTime);
} catch (OperationCanceledException oce) {
node.Value.TryCancel(oce, _walker.Version);
node.Skip();
LogCanceled(node.Value.Module);
} catch (Exception exception) {
node.Value.TrySetException(exception, _walker.Version);
node.Commit();
LogException(node.Value.Module, exception);
} finally {
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled)
{
_progress.ReportRemaining(_walker.Remaining);
}
Interlocked.Decrement(ref _runningTasks);
ace?.Signal();
}
}
private void AnalyzeLoop(IDependencyChainLoopNode <PythonAnalyzerEntry> loopNode, Stopwatch stopWatch)
{
var version = _walker.Version;
var entries = new Dictionary <AnalysisModuleKey, (IPythonModule Module, PythonAnalyzerEntry Entry)>();
var variables = new Dictionary <(AnalysisModuleKey Module, string Name), int>();
var importNames = new List <(AnalysisModuleKey From, int FromPosition, AnalysisModuleKey To, string ToName)>();
var cachedVariables = new Dictionary <AnalysisModuleKey, IVariableCollection>();
var asts = new Dictionary <AnalysisModuleKey, PythonAst>();
var startTime = stopWatch.Elapsed;
// Note: loop analysis is not cancellable. The reason is that when smaller loop
// appears inside a larger loop gets canceled, it will not be re-walked during
// the outer loop analysis. For example, functools <=> _functools loop and
// the related CircularDependencyFunctools test.
foreach (var entry in loopNode.Values)
{
ActivityTracker.OnEnqueueModule(entry.Module.FilePath);
if (!CanUpdateAnalysis(entry, Version, out var module, out var ast))
{
_log?.Log(TraceEventType.Verbose, $"Analysis of loop canceled.");
return;
}
var moduleKey = new AnalysisModuleKey(module);
entries[moduleKey] = (module, entry);
var analysis = _analyzer.TryRestoreCachedAnalysis(module);
if (analysis != null)
{
AddLoopImportsFromCachedAnalysis(importNames, variables, moduleKey, analysis);
cachedVariables.Add(moduleKey, analysis.GlobalScope.Variables);
}
else
{
AddLoopImportsFromAst(importNames, variables, moduleKey, ast);
asts.Add(moduleKey, ast);
}
}
if (asts.Count == 0)
{
// Fully cached loop
if (_log != null && _log.LogLevel == TraceEventType.Verbose)
{
var names = string.Join(", ", cachedVariables.Select(v => v.Key.Name));
_log?.Log(TraceEventType.Verbose, $"Fully cached modules cycle: {names}");
}
return;
}
var imports = new List <(AnalysisModuleKey From, int FromPosition, AnalysisModuleKey To, int ToPosition)>();
foreach (var(fromModule, fromPosition, toModule, toName) in importNames)
{
if (!entries.ContainsKey(toModule))
{
continue;
}
if (toName == null)
{
imports.Add((fromModule, fromPosition, toModule, 0));
}
else if (variables.TryGetValue((toModule, toName), out var toPosition))
{
imports.Add((fromModule, fromPosition, toModule, toPosition));
}
}
var startingKeys = LocationLoopResolver <AnalysisModuleKey> .FindStartingItems(imports);
var variableHandler = new LoopImportedVariableHandler(_services, asts, cachedVariables, () => false);
foreach (var key in startingKeys)
{
if (asts.TryGetValue(key, out var startingAst) && entries.TryGetValue(key, out var me))
{
variableHandler.WalkModule(me.Module, startingAst);
}
}
foreach (var walker in variableHandler.Walkers)
{
asts.Remove(new AnalysisModuleKey(walker.Module));
}
while (asts.Count > 0)
{
var(moduleKey, ast) = asts.First();
variableHandler.WalkModule(entries[moduleKey].Module, ast);
foreach (var walker in variableHandler.Walkers)
{
asts.Remove(new AnalysisModuleKey(walker.Module));
}
}
foreach (var walker in variableHandler.Walkers)
{
var module = (IDocument)walker.Module;
var moduleKey = new AnalysisModuleKey(module);
if (entries.TryGetValue(moduleKey, out var e))
{
var analysis = CreateAnalysis(null, module, walker.Ast, version, walker);
CompleteAnalysis(e.Entry, module, version, analysis);
}
}
loopNode.MarkWalked();
LogCompleted(loopNode, entries.Values.Select(v => v.Module), stopWatch, startTime);
}