protected virtual void CheckMemoryUsage()
{
if (_memoryManager == null ||
_crawlContext.IsCrawlHardStopRequested ||
_crawlContext.CrawlConfiguration.MaxMemoryUsageInMb < 1)
{
return;
}
int currentMemoryUsage = _memoryManager.GetCurrentUsageInMb();
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Current memory usage for site [{0}] is [{1}mb]", _crawlContext.RootUri, currentMemoryUsage);
}
if (currentMemoryUsage > _crawlContext.CrawlConfiguration.MaxMemoryUsageInMb)
{
_memoryManager.Dispose();
_memoryManager = null;
string message = string.Format("Process is using [{0}mb] of memory which is above the max configured of [{1}mb] for site [{2}]. This is configurable through the maxMemoryUsageInMb in app.conf or CrawlConfiguration.MaxMemoryUsageInMb.", currentMemoryUsage, _crawlContext.CrawlConfiguration.MaxMemoryUsageInMb, _crawlContext.RootUri);
_crawlResult.ErrorException = new InsufficientMemoryException(message);
_logger.Fatal(_crawlResult.ErrorException);
_crawlContext.IsCrawlHardStopRequested = true;
}
}
protected virtual void CheckMemoryUsage()
{
if (_memoryManager == null ||
_crawlContext.IsCrawlHardStopRequested ||
_crawlContext.CrawlConfiguration.MaxMemoryUsageInMb < 1)
{
return;
}
var currentMemoryUsage = _memoryManager.GetCurrentUsageInMb();
_logger.LogDebug($"Current memory usage for site [{_crawlContext.RootUri}] is [{currentMemoryUsage}mb]");
if (currentMemoryUsage > _crawlContext.CrawlConfiguration.MaxMemoryUsageInMb)
{
_memoryManager.Dispose();
_memoryManager = null;
string message = $"Process is using [{currentMemoryUsage}mb] of memory which is above the max configured of [{_crawlContext.CrawlConfiguration.MaxMemoryUsageInMb}mb] for site [{_crawlContext.RootUri}]. This is configurable through the maxMemoryUsageInMb in app.conf or CrawlConfiguration.MaxMemoryUsageInMb.";
_crawlResult.ErrorException = new InsufficientMemoryException(message);
_logger.LogCritical("Memory exception", _crawlResult.ErrorException);
_crawlContext.IsCrawlHardStopRequested = true;
}
}
/// <inheritdoc />
public virtual void Dispose()
{
_threadManager?.Dispose();
_scheduler?.Dispose();
_pageRequester?.Dispose();
_memoryManager?.Dispose();
}
public void Dispose()
{
memoryManager.Dispose();
parcelScenesCleaner.Dispose();
cullingController.Dispose();
debugController.Dispose();
}
protected void Dispose(bool isDisposing)
{
if (isDisposing)
{
_memoryManager.Dispose();
}
// Dispose unmanaged resources.
}
public void Dispose()
{
memoryManager.Dispose();
parcelScenesCleaner.Dispose();
cullingController.Dispose();
debugController.Dispose();
webRequest.Dispose();
serviceProviders.Dispose();
}
/// <inheritdoc />
public virtual void Dispose()
{
if (_threadManager != null)
{
_threadManager.Dispose();
}
if (_scheduler != null)
{
_scheduler.Dispose();
}
if (_pageRequester != null)
{
_pageRequester.Dispose();
}
if (_memoryManager != null)
{
_memoryManager.Dispose();
}
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
_memoryManager.Dispose();
}
disposedValue = true;
}
}
public override void OnTeardown()
{
_manager.Dispose();
}
public void Dispose()
{
_memoryManager?.Dispose();
GC.SuppressFinalize(this);
}
public void Dispose()
{
_memoryManager.Dispose();
}
