public PythonAnalyzerSession(IServiceContainer services,
IProgressReporter progress,
Action <Task> startNextSession,
CancellationToken analyzerCancellationToken,
IDependencyChainWalker <AnalysisModuleKey, PythonAnalyzerEntry> walker,
int version,
PythonAnalyzerEntry entry,
bool forceGC = false)
{
_services = services;
_startNextSession = startNextSession;
_analyzerCancellationToken = analyzerCancellationToken;
Version = version;
AffectedEntriesCount = walker?.AffectedValues.Count ?? 1;
_walker = walker;
_entry = entry;
_state = State.NotStarted;
_forceGC = forceGC;
_diagnosticsService = _services.GetService <IDiagnosticsService>();
_platformService = _services.GetService <IOSPlatform>();
_analyzer = _services.GetService <IPythonAnalyzer>();
_log = _services.GetService <ILogger>();
_moduleDatabaseService = _services.GetService <IModuleDatabaseService>();
_progress = progress;
var interpreter = _services.GetService <IPythonInterpreter>();
_modulesPathResolver = interpreter.ModuleResolution.CurrentPathResolver;
_typeshedPathResolver = interpreter.TypeshedResolution.CurrentPathResolver;
}
public PythonAnalyzerSession(IServiceManager services,
IProgressReporter progress,
AsyncManualResetEvent analysisCompleteEvent,
Action <Task> startNextSession,
CancellationToken analyzerCancellationToken,
IDependencyChainWalker <AnalysisModuleKey, PythonAnalyzerEntry> walker,
int version,
PythonAnalyzerEntry entry)
{
_services = services;
_analysisCompleteEvent = analysisCompleteEvent;
_startNextSession = startNextSession;
_analyzerCancellationToken = analyzerCancellationToken;
Version = version;
AffectedEntriesCount = walker?.AffectedValues.Count ?? 1;
_walker = walker;
_entry = entry;
_state = State.NotStarted;
_diagnosticsService = _services.GetService <IDiagnosticsService>();
_analyzer = _services.GetService <IPythonAnalyzer>();
_log = _services.GetService <ILogger>();
_telemetry = _services.GetService <ITelemetryService>();
_progress = progress;
}
private async Task StartAsync()
{
_progress.ReportRemaining(_walker.Remaining);
lock (_syncObj) {
var notAnalyzed = _walker.AffectedValues.Count(e => e.NotAnalyzed);
if (_isCanceled && notAnalyzed < _maxTaskRunning)
{
_state = State.Completed;
return;
}
}
var stopWatch = Stopwatch.StartNew();
foreach (var affectedEntry in _walker.AffectedValues)
{
affectedEntry.Invalidate(Version);
}
var originalRemaining = _walker.Remaining;
var remaining = originalRemaining;
try {
_log?.Log(TraceEventType.Verbose, $"Analysis version {Version} of {originalRemaining} entries has started.");
remaining = await AnalyzeAffectedEntriesAsync(stopWatch);
} finally {
stopWatch.Stop();
bool isCanceled;
bool isFinal;
lock (_syncObj) {
isCanceled = _isCanceled;
_state = State.Completed;
isFinal = _walker.MissingKeys.Count == 0 && !isCanceled && remaining == 0;
_walker = null;
}
if (!isCanceled)
{
_progress.ReportRemaining(remaining);
if (isFinal)
{
var(modulesCount, totalMilliseconds) = ActivityTracker.EndTracking();
(_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds);
_log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms.");
}
}
}
var elapsed = stopWatch.Elapsed.TotalMilliseconds;
LogResults(_log, elapsed, originalRemaining, remaining, Version);
ForceGCIfNeeded(originalRemaining, remaining);
}
private async Task StartAsync()
{
_progress.ReportRemaining(_walker.Remaining);
lock (_syncObj) {
var notAnalyzed = _walker.AffectedValues.Count(e => e.NotAnalyzed);
if (_isCanceled && notAnalyzed < _maxTaskRunning)
{
_state = State.Completed;
return;
}
}
var stopWatch = Stopwatch.StartNew();
var originalRemaining = _walker.Remaining;
var remaining = originalRemaining;
try {
_log?.Log(TraceEventType.Verbose, $"Analysis version {Version} of {originalRemaining} entries has started.");
remaining = await AnalyzeAffectedEntriesAsync(stopWatch);
} finally {
stopWatch.Stop();
bool isCanceled;
bool isFinal;
lock (_syncObj) {
isCanceled = _isCanceled;
_state = State.Completed;
isFinal = _walker.MissingKeys.Count == 0 && !isCanceled && remaining == 0;
_walker = null;
}
if (!isCanceled)
{
_progress.ReportRemaining(remaining);
if (isFinal)
{
var(modulesCount, totalMilliseconds) = ActivityTracker.EndTracking();
totalMilliseconds = Math.Round(totalMilliseconds, 2);
(_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds);
_log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms.");
//#if DEBUG
// var notReady = _analyzer.LoadedModules.Where(m => (m.ModuleType == ModuleType.Library || m.ModuleType == ModuleType.Stub) && m.Analysis is EmptyAnalysis).ToArray();
// Debug.Assert(notReady.Length == 0);
//#endif
}
}
}
var elapsed = stopWatch.Elapsed.TotalMilliseconds;
LogResults(_log, elapsed, originalRemaining, remaining, Version);
ForceGCIfNeeded(_log, originalRemaining, remaining, _forceGC);
}
private async Task StartAsync()
{
_progress.ReportRemaining(_walker.Remaining);
lock (_syncObj) {
var notAnalyzed = _walker.AffectedValues.Count(e => e.NotAnalyzed);
if (_isCanceled && notAnalyzed < _maxTaskRunning)
{
_state = State.Completed;
return;
}
}
var stopWatch = Stopwatch.StartNew();
var originalRemaining = _walker.Remaining;
var remaining = originalRemaining;
try {
_log?.Log(TraceEventType.Verbose, $"Analysis version {Version} of {originalRemaining} entries has started.");
remaining = await AnalyzeAffectedEntriesAsync(stopWatch);
Debug.Assert(_ace.Count == 0);
} finally {
stopWatch.Stop();
var isFinal = false;
lock (_syncObj) {
if (!_isCanceled)
{
_progress.ReportRemaining(remaining);
}
_state = State.Completed;
isFinal = _walker.MissingKeys.Count == 0 && !_isCanceled && remaining == 0;
_walker = null;
}
if (isFinal)
{
var(modulesCount, totalMilliseconds) = ActivityTracker.EndTracking();
totalMilliseconds = Math.Round(totalMilliseconds, 2);
if (await _analyzer.RaiseAnalysisCompleteAsync(modulesCount, totalMilliseconds))
{
_log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms.");
}
}
}
var elapsed = stopWatch.Elapsed.TotalMilliseconds;
LogResults(_log, elapsed, originalRemaining, remaining, Version);
ForceGCIfNeeded(_log, originalRemaining, remaining, _forceGC);
}
private async Task StartAsync(IDependencyChainWalker <AnalysisModuleKey, PythonAnalyzerEntry> walker)
{
_progress.ReportRemaining(walker.Remaining);
lock (_syncObj) {
var notAnalyzed = walker.AffectedValues.Count(e => e.NotAnalyzed);
if (_isCanceled && notAnalyzed < _maxTaskRunning)
{
_state = State.Completed;
_cts.Dispose();
return;
}
}
var cancellationToken = _cts.Token;
var stopWatch = Stopwatch.StartNew();
foreach (var affectedEntry in walker.AffectedValues)
{
affectedEntry.Invalidate(Version);
}
var originalRemaining = walker.Remaining;
var remaining = originalRemaining;
try {
_log?.Log(TraceEventType.Verbose, $"Analysis version {walker.Version} of {originalRemaining} entries has started.");
remaining = await AnalyzeAffectedEntriesAsync(walker, stopWatch, cancellationToken);
} finally {
_cts.Dispose();
stopWatch.Stop();
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
_state = State.Completed;
}
if (!isCanceled)
{
_progress.ReportRemaining(remaining);
}
}
var elapsed = stopWatch.Elapsed.TotalMilliseconds;
SendTelemetry(elapsed, originalRemaining, remaining, walker.Version);
LogResults(elapsed, originalRemaining, remaining, walker.Version);
}
private async Task StartAsync()
{
_progress.ReportRemaining(_walker.Remaining);
lock (_syncObj) {
var notAnalyzed = _walker.AffectedValues.Count(e => e.NotAnalyzed);
if (_isCanceled && notAnalyzed < _maxTaskRunning)
{
_state = State.Completed;
return;
}
}
var stopWatch = Stopwatch.StartNew();
foreach (var affectedEntry in _walker.AffectedValues)
{
affectedEntry.Invalidate(Version);
}
var originalRemaining = _walker.Remaining;
var remaining = originalRemaining;
try {
_log?.Log(TraceEventType.Verbose, $"Analysis version {Version} of {originalRemaining} entries has started.");
remaining = await AnalyzeAffectedEntriesAsync(stopWatch);
} finally {
stopWatch.Stop();
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
_state = State.Completed;
_walker = null;
}
if (!isCanceled)
{
_progress.ReportRemaining(remaining);
}
}
var elapsed = stopWatch.Elapsed.TotalMilliseconds;
SendTelemetry(_telemetry, elapsed, originalRemaining, remaining, Version);
LogResults(_log, elapsed, originalRemaining, remaining, Version);
ForceGCIfNeeded(originalRemaining, remaining);
}
private async Task <int> AnalyzeAffectedEntriesAsync(IDependencyChainWalker <AnalysisModuleKey, PythonAnalyzerEntry> walker, Stopwatch stopWatch, CancellationToken cancellationToken)
{
IDependencyChainNode <PythonAnalyzerEntry> node;
var remaining = 0;
while ((node = await walker.GetNextAsync(cancellationToken)) != null)
{
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (isCanceled && !node.Value.NotAnalyzed)
{
remaining++;
node.Skip();
continue;
}
if (Interlocked.Increment(ref _runningTasks) >= _maxTaskRunning || walker.Remaining == 1)
{
Analyze(walker, node, stopWatch, cancellationToken);
}
else
{
StartAnalysis(walker, node, stopWatch, cancellationToken).DoNotWait();
}
}
if (walker.MissingKeys.All(k => k.IsTypeshed))
{
Interlocked.Exchange(ref _runningTasks, 0);
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled)
{
_analysisCompleteEvent.Set();
}
}
return(remaining);
}
public bool TryCreateWalker(int version, out IDependencyChainWalker <TKey, TValue> walker)
{
DependencyGraphSnapshot <TKey, TValue> snapshot;
ImmutableArray <DependencyVertex <TKey, TValue> > changedVertices;
lock (_syncObj) {
snapshot = _vertices.Snapshot;
if (version != snapshot.Version)
{
walker = default;
return(false);
}
changedVertices = ImmutableArray <DependencyVertex <TKey, TValue> > .Create(_changedVertices.Values);
}
walker = CreateDependencyChainWalker(this, snapshot, changedVertices);
return(true);
}
/// <summary>
/// Performs analysis of the document. Returns document global scope
/// with declared variables and inner scopes. Does not analyze chain
/// of dependencies, it is intended for the single file analysis.
/// </summary>
private void Analyze(IDependencyChainWalker <AnalysisModuleKey, PythonAnalyzerEntry> walker, IDependencyChainNode <PythonAnalyzerEntry> node, Stopwatch stopWatch, CancellationToken cancellationToken)
{
IPythonModule module;
try {
var entry = node.Value;
if (!entry.IsValidVersion(walker.Version, out module, out var ast))
{
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
node.Skip();
return;
}
var startTime = stopWatch.Elapsed;
AnalyzeEntry(entry, module, ast, walker.Version, cancellationToken);
node.Commit();
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms.");
} catch (OperationCanceledException oce) {
node.Value.TryCancel(oce, walker.Version);
node.Skip();
module = node.Value.Module;
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
} catch (Exception exception) {
module = node.Value.Module;
node.Value.TrySetException(exception, walker.Version);
node.Commit();
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed.");
} finally {
bool isCanceled;
lock (_syncObj) {
isCanceled = _isCanceled;
}
if (!isCanceled)
{
_progress.ReportRemaining(walker.Remaining);
}
Interlocked.Decrement(ref _runningTasks);
}
}
private Task StartAnalysis(IDependencyChainWalker <AnalysisModuleKey, PythonAnalyzerEntry> walker, IDependencyChainNode <PythonAnalyzerEntry> node, Stopwatch stopWatch, CancellationToken cancellationToken) => Task.Run(() => Analyze(walker, node, stopWatch, cancellationToken), cancellationToken);
