예제 #1
0
        protected virtual void CheckMemoryUsage()
        {
            if (_memoryManager == null ||
                _crawlContext.IsCrawlHardStopRequested ||
                _crawlContext.CrawlConfiguration.MaxMemoryUsageInMb < 1)
            {
                return;
            }

            int currentMemoryUsage = _memoryManager.GetCurrentUsageInMb();

            if (_logger.IsDebugEnabled)
            {
                _logger.DebugFormat("Current memory usage for site [{0}] is [{1}mb]", _crawlContext.RootUri, currentMemoryUsage);
            }

            if (currentMemoryUsage > _crawlContext.CrawlConfiguration.MaxMemoryUsageInMb)
            {
                _memoryManager.Dispose();
                _memoryManager = null;

                string message = string.Format("Process is using [{0}mb] of memory which is above the max configured of [{1}mb] for site [{2}]. This is configurable through the maxMemoryUsageInMb in app.conf or CrawlConfiguration.MaxMemoryUsageInMb.", currentMemoryUsage, _crawlContext.CrawlConfiguration.MaxMemoryUsageInMb, _crawlContext.RootUri);
                _crawlResult.ErrorException = new InsufficientMemoryException(message);

                _logger.Fatal(_crawlResult.ErrorException);
                _crawlContext.IsCrawlHardStopRequested = true;
            }
        }
예제 #2
0
        protected virtual void CheckMemoryUsage()
        {
            if (_memoryManager == null ||
                _crawlContext.IsCrawlHardStopRequested ||
                _crawlContext.CrawlConfiguration.MaxMemoryUsageInMb < 1)
            {
                return;
            }

            var currentMemoryUsage = _memoryManager.GetCurrentUsageInMb();

            _logger.LogDebug($"Current memory usage for site [{_crawlContext.RootUri}] is [{currentMemoryUsage}mb]");

            if (currentMemoryUsage > _crawlContext.CrawlConfiguration.MaxMemoryUsageInMb)
            {
                _memoryManager.Dispose();
                _memoryManager = null;

                string message = $"Process is using [{currentMemoryUsage}mb] of memory which is above the max configured of [{_crawlContext.CrawlConfiguration.MaxMemoryUsageInMb}mb] for site [{_crawlContext.RootUri}]. This is configurable through the maxMemoryUsageInMb in app.conf or CrawlConfiguration.MaxMemoryUsageInMb.";
                _crawlResult.ErrorException = new InsufficientMemoryException(message);

                _logger.LogCritical("Memory exception", _crawlResult.ErrorException);
                _crawlContext.IsCrawlHardStopRequested = true;
            }
        }
예제 #3
0
 /// <inheritdoc />
 public virtual void Dispose()
 {
     _threadManager?.Dispose();
     _scheduler?.Dispose();
     _pageRequester?.Dispose();
     _memoryManager?.Dispose();
 }
예제 #4
0
 public void Dispose()
 {
     memoryManager.Dispose();
     parcelScenesCleaner.Dispose();
     cullingController.Dispose();
     debugController.Dispose();
 }
        protected void Dispose(bool isDisposing)
        {
            if (isDisposing)
            {
                _memoryManager.Dispose();
            }

            // Dispose unmanaged resources.
        }
예제 #6
0
 public void Dispose()
 {
     memoryManager.Dispose();
     parcelScenesCleaner.Dispose();
     cullingController.Dispose();
     debugController.Dispose();
     webRequest.Dispose();
     serviceProviders.Dispose();
 }
예제 #7
0
 /// <inheritdoc />
 public virtual void Dispose()
 {
     if (_threadManager != null)
     {
         _threadManager.Dispose();
     }
     if (_scheduler != null)
     {
         _scheduler.Dispose();
     }
     if (_pageRequester != null)
     {
         _pageRequester.Dispose();
     }
     if (_memoryManager != null)
     {
         _memoryManager.Dispose();
     }
 }
예제 #8
0
 protected virtual void Dispose(bool disposing)
 {
     if (!disposedValue)
     {
         if (disposing)
         {
             _memoryManager.Dispose();
         }
         disposedValue = true;
     }
 }
예제 #9
0
 public override void OnTeardown()
 {
     _manager.Dispose();
 }
예제 #10
0
 public void Dispose()
 {
     _memoryManager?.Dispose();
     GC.SuppressFinalize(this);
 }
예제 #11
0
 public void Dispose()
 {
     _memoryManager.Dispose();
 }