예제 #1
0
        public PeriodicReader(ILogger <PeriodicReader> logger,
                              IConfiguration configuration,
                              IHardwareMonitor hardwareMonitor,
                              ISensorCacheMutator sensorCacheMutator)
        {
            _logger      = logger;
            _periodicity = configuration.GetValue <int>(ConfigurationConstants.PeriodicityKey);

            _hardwareMonitor    = hardwareMonitor;
            _sensorCacheMutator = sensorCacheMutator;
        }
예제 #2
0
        public RendererPoolBlock(IHardwareMonitor hardwareMonitor, Func <IHtmlRenderer> getHtmlRenderer, ILog log)
        {
            _log             = log;
            _hardwareMonitor = hardwareMonitor;

            _counter = new Counter();
            Events   = new BufferBlock <Event>(new DataflowBlockOptions {
                EnsureOrdered = true
            });

            CreateHtmlRenderer();
            CreateAndDestroyHtmlRenderersAdaptively();

            #region Local Functions

            void CreateHtmlRenderer()
            {
                if (!this.Post(getHtmlRenderer()))
                {
                    log.Info($"Failed to post newly created {nameof(HtmlRenderer)} to {nameof(RendererPoolBlock)}.");
                }

                _counter.CreatedHtmlRenderCount++;
            }

            void CreateAndDestroyHtmlRenderersAdaptively()
            {
                hardwareMonitor.OnLowCpuAndMemoryUsage += (averageCpuUsage, memoryUsage) =>
                {
                    if (OutputCount > 0 || _counter.CreatedHtmlRenderCount == Configurations.MaxHtmlRendererCount)
                    {
                        return;
                    }
                    try
                    {
                        CreateHtmlRenderer();
                        log.Info(
                            $"Low CPU usage ({averageCpuUsage}%) and low memory usage ({memoryUsage}%) detected. " +
                            $"Browser count increased from {_counter.CreatedHtmlRenderCount - 1} to {_counter.CreatedHtmlRenderCount}."
                            );
                    }
                    catch (Exception exception)
                    {
                        log.Error($"Failed to create {nameof(HtmlRenderer)}.", exception);
                    }
                };
                hardwareMonitor.OnHighCpuOrMemoryUsage += (averageCpuUsage, memoryUsage) =>
                {
                    if (_counter.CreatedHtmlRenderCount == 1)
                    {
                        return;
                    }

                    this.Receive().Dispose();
                    _counter.CreatedHtmlRenderCount--;

                    if (averageCpuUsage == null && memoryUsage == null)
                    {
                        throw new ArgumentException(nameof(averageCpuUsage), nameof(memoryUsage));
                    }

                    if (averageCpuUsage != null && memoryUsage != null)
                    {
                        log.Info(
                            $"High CPU usage ({averageCpuUsage}%) and high memory usage ({memoryUsage}%) detected. " +
                            $"Browser count decreased from {_counter.CreatedHtmlRenderCount + 1} to {_counter.CreatedHtmlRenderCount}."
                            );
                    }
                    else if (averageCpuUsage != null)
                    {
                        log.Info(
                            $"High CPU usage ({averageCpuUsage}%) detected. " +
                            $"Browser count decreased from {_counter.CreatedHtmlRenderCount + 1} to {_counter.CreatedHtmlRenderCount}."
                            );
                    }
                    else
                    {
                        log.Info(
                            $"High memory usage ({memoryUsage}%) detected. " +
                            $"Browser count decreased from {_counter.CreatedHtmlRenderCount + 1} to {_counter.CreatedHtmlRenderCount}."
                            );
                    }
                };
            }

            #endregion
        }
예제 #3
0
        public NetworkServicePool(Configurations configurations, IEventBroadcaster eventBroadcaster, IHardwareMonitor hardwareMonitor,
                                  IMemory memory, ILog log, Func <IResourceExtractor> getResourceExtractor, Func <IResourceVerifier> getResourceVerifier,
                                  Func <IHtmlRenderer> getHtmlRenderer)
        {
            _log                   = log;
            _objectDisposed        = false;
            _statistics            = new Statistics();
            _eventBroadcaster      = eventBroadcaster;
            _resourceExtractorPool = new BlockingCollection <IResourceExtractor>();
            _resourceVerifierPool  = new BlockingCollection <IResourceVerifier>();
            _htmlRendererPool      = new BlockingCollection <IHtmlRenderer>();

            PreCreateServices();
            CreateAndDestroyHtmlRenderersAdaptively();

            void PreCreateServices()
            {
                PreCreateResourceExtractors();
                PreCreateResourceVerifiers();
                CreateHtmlRenderer();

                void PreCreateResourceExtractors()
                {
                    for (var resourceExtractorId = 0; resourceExtractorId < configurations.ResourceExtractorCount; resourceExtractorId++)
                    {
                        var resourceExtractor = getResourceExtractor();
                        _resourceExtractorPool.Add(resourceExtractor);
                        _statistics.CreatedResourceExtractorCount++;
                    }
                }

                void PreCreateResourceVerifiers()
                {
                    for (var resourceVerifierId = 0; resourceVerifierId < configurations.ResourceVerifierCount; resourceVerifierId++)
                    {
                        var resourceVerifier = getResourceVerifier();
                        _resourceVerifierPool.Add(resourceVerifier);
                        _statistics.CreatedResourceVerifierCount++;
                    }
                }
            }

            void CreateAndDestroyHtmlRenderersAdaptively()
            {
                hardwareMonitor.OnLowCpuAndMemoryUsage += (averageCpuUsage, memoryUsage) =>
                {
                    if (_htmlRendererPool.Count > 0 || _statistics.CreatedHtmlRendererCount == configurations.MaxHtmlRendererCount)
                    {
                        return;
                    }
                    CreateHtmlRenderer();

                    var createdHtmlRendererCount = _statistics.CreatedHtmlRendererCount;
                    _log.Info(
                        $"Low CPU usage ({averageCpuUsage}%) and low memory usage ({memoryUsage}%) detected. " +
                        $"Browser count increased from {createdHtmlRendererCount - 1} to {createdHtmlRendererCount}."
                        );
                };
                hardwareMonitor.OnHighCpuOrMemoryUsage += (averageCpuUsage, memoryUsage) =>
                {
                    if (_statistics.CreatedHtmlRendererCount == 1)
                    {
                        return;
                    }
                    _htmlRendererPool.Take().Dispose();
                    _statistics.CreatedHtmlRendererCount--;

                    if (averageCpuUsage == null && memoryUsage == null)
                    {
                        throw new ArgumentException(nameof(averageCpuUsage), nameof(memoryUsage));
                    }

                    var createdHtmlRendererCount = _statistics.CreatedHtmlRendererCount;
                    if (averageCpuUsage != null && memoryUsage != null)
                    {
                        _log.Info(
                            $"High CPU usage ({averageCpuUsage}%) and high memory usage ({memoryUsage}%) detected. " +
                            $"Browser count decreased from {createdHtmlRendererCount + 1} to {createdHtmlRendererCount}."
                            );
                    }
                    else if (averageCpuUsage != null)
                    {
                        _log.Info(
                            $"High CPU usage ({averageCpuUsage}%) detected. " +
                            $"Browser count decreased from {createdHtmlRendererCount + 1} to {createdHtmlRendererCount}."
                            );
                    }
                    else
                    {
                        _log.Info(
                            $"High memory usage ({memoryUsage}%) detected. " +
                            $"Browser count decreased from {createdHtmlRendererCount + 1} to {createdHtmlRendererCount}."
                            );
                    }
                };
            }

            void CreateHtmlRenderer()
            {
                var htmlRenderer = getHtmlRenderer();

                htmlRenderer.OnResourceCaptured += memory.MemorizeToBeVerifiedResource;
                _htmlRendererPool.Add(htmlRenderer);
                _statistics.CreatedHtmlRendererCount++;
            }
        }
예제 #4
0
        public BrokenLinkCollectionWorkflow(CancellationToken cancellationToken, Configurations configurations, IStatistics statistics,
                                            IHardwareMonitor hardwareMonitor, IResourceExtractor resourceExtractor, IReportWriter reportWriter, ILog log,
                                            IResourceEnricher resourceEnricher, IResourceVerifier resourceVerifier, Func <IHtmlRenderer> getHtmlRenderer)
        {
            _log = log;
            _coordinatorBlock               = new CoordinatorBlock(cancellationToken, log);
            _eventBroadcasterBlock          = new EventBroadcasterBlock(cancellationToken);
            _processingResultGeneratorBlock = new ProcessingResultGeneratorBlock(cancellationToken, resourceExtractor, log);
            _reportWriterBlock              = new ReportWriterBlock(cancellationToken, reportWriter, log);
            _resourceEnricherBlock          = new ResourceEnricherBlock(cancellationToken, resourceEnricher, log);
            _resourceVerifierBlock          = new ResourceVerifierBlock(cancellationToken, statistics, resourceVerifier, log);
            _htmlRendererBlock              = new HtmlRendererBlock(
                cancellationToken,
                statistics,
                log,
                configurations,
                hardwareMonitor,
                getHtmlRenderer
                );

            WireUpBlocks();

            void WireUpBlocks()
            {
                var generalDataflowLinkOptions = new DataflowLinkOptions {
                    PropagateCompletion = true
                };

                _coordinatorBlock.LinkTo(NullTarget <Resource>(), PropagateNullObjectsOnly <Resource>());
                _coordinatorBlock.LinkTo(_resourceEnricherBlock, generalDataflowLinkOptions);

                _resourceEnricherBlock.LinkTo(NullTarget <Resource>(), PropagateNullObjectsOnly <Resource>());
                _resourceEnricherBlock.LinkTo(_resourceVerifierBlock, generalDataflowLinkOptions);
                _resourceEnricherBlock.FailedProcessingResults.LinkTo(_coordinatorBlock);

                _resourceVerifierBlock.LinkTo(NullTarget <Resource>(), PropagateNullObjectsOnly <Resource>());
                _resourceVerifierBlock.LinkTo(_htmlRendererBlock, generalDataflowLinkOptions);
                _resourceVerifierBlock.FailedProcessingResults.LinkTo(_coordinatorBlock);
                _resourceVerifierBlock.VerificationResults.LinkTo(_reportWriterBlock);
                _resourceVerifierBlock.Events.LinkTo(_eventBroadcasterBlock);

                _htmlRendererBlock.LinkTo(NullTarget <RenderingResult>(), PropagateNullObjectsOnly <RenderingResult>());
                _htmlRendererBlock.LinkTo(_processingResultGeneratorBlock, generalDataflowLinkOptions);
                _htmlRendererBlock.VerificationResults.LinkTo(_reportWriterBlock, generalDataflowLinkOptions);
                _htmlRendererBlock.FailedProcessingResults.LinkTo(_coordinatorBlock);
                _htmlRendererBlock.Events.LinkTo(_eventBroadcasterBlock, generalDataflowLinkOptions);

                _processingResultGeneratorBlock.LinkTo(NullTarget <ProcessingResult>(), PropagateNullObjectsOnly <ProcessingResult>());
                _processingResultGeneratorBlock.LinkTo(_coordinatorBlock);

                _eventBroadcasterBlock.LinkTo(NullTarget <Event>(), PropagateNullObjectsOnly <Event>());

                Predicate <T> PropagateNullObjectsOnly <T>()
                {
                    return(@object => @object == null);
                }

                ITargetBlock <T> NullTarget <T>()
                {
                    return(DataflowBlock.NullTarget <T>());
                }
            }
        }
예제 #5
0
        public HtmlRendererBlock(CancellationToken cancellationToken, IStatistics statistics, ILog log, Configurations configurations,
                                 IHardwareMonitor hardwareMonitor, Func <IHtmlRenderer> getHtmlRenderer) : base(cancellationToken, maxDegreeOfParallelism: 300)
        {
            _log               = log;
            _statistics        = statistics;
            _cancellationToken = cancellationToken;
            _htmlRenderers     = new BlockingCollection <IHtmlRenderer>();
            (int createdHtmlRenderCount, int disposedHtmlRendererCount)counter = (0, 0);

            var generalDataflowBlockOptions = new DataflowBlockOptions {
                CancellationToken = cancellationToken
            };

            VerificationResults     = new BufferBlock <VerificationResult>(generalDataflowBlockOptions);
            FailedProcessingResults = new BufferBlock <FailedProcessingResult>(generalDataflowBlockOptions);
            Events = new BufferBlock <Event>(new DataflowBlockOptions {
                EnsureOrdered = true, CancellationToken = cancellationToken
            });

            base.Completion.ContinueWith(_ =>
            {
                FailedProcessingResults.Complete();
                VerificationResults.Complete();
                DisposeHtmlRenderers();
                Events.Complete();
                CheckMemoryLeak();

                void CheckMemoryLeak()
                {
                    if (counter.disposedHtmlRendererCount == counter.createdHtmlRenderCount)
                    {
                        return;
                    }
                    var resourceName      = $"{nameof(HtmlRenderer)}{(counter.createdHtmlRenderCount > 1 ? "s" : string.Empty)}";
                    var disposedCountText = counter.disposedHtmlRendererCount == 0 ? "none" : $"only {counter.disposedHtmlRendererCount}";
                    _log.Warn(
                        "Orphaned resources detected! " +
                        $"{counter.createdHtmlRenderCount} {resourceName} were created but {disposedCountText} could be found and disposed."
                        );
                }
                void DisposeHtmlRenderers()
                {
                    while (_htmlRenderers.Any())
                    {
                        _htmlRenderers.Take().Dispose();
                        counter.disposedHtmlRendererCount++;

                        var webBrowserClosedEvent = new Event
                        {
                            EventType = EventType.StopProgressUpdated,
                            Message   = $"Closing web browsers ({counter.disposedHtmlRendererCount}/{counter.createdHtmlRenderCount}) ..."
                        };
                        if (!Events.Post(webBrowserClosedEvent))
                        {
                            _log.Error($"Failed to post data to buffer block named [{nameof(Events)}].");
                        }
                    }
                    _htmlRenderers?.Dispose();
                }
            });

            CreateHtmlRenderer();
            CreateAndDestroyHtmlRenderersAdaptively();

            void CreateHtmlRenderer()
            {
                _htmlRenderers.Add(getHtmlRenderer(), CancellationToken.None);
                counter.createdHtmlRenderCount++;
            }

            void CreateAndDestroyHtmlRenderersAdaptively()
            {
                hardwareMonitor.OnLowCpuAndMemoryUsage += (averageCpuUsage, memoryUsage) =>
                {
                    if (_htmlRenderers.Count > 0 || counter.createdHtmlRenderCount == configurations.MaxHtmlRendererCount)
                    {
                        return;
                    }
                    CreateHtmlRenderer();

                    _log.Info(
                        $"Low CPU usage ({averageCpuUsage}%) and low memory usage ({memoryUsage}%) detected. " +
                        $"Browser count increased from {counter.createdHtmlRenderCount - 1} to {counter.createdHtmlRenderCount}."
                        );
                };
                hardwareMonitor.OnHighCpuOrMemoryUsage += (averageCpuUsage, memoryUsage) =>
                {
                    if (counter.createdHtmlRenderCount == 1)
                    {
                        return;
                    }
                    _htmlRenderers.Take().Dispose();
                    counter.createdHtmlRenderCount--;

                    if (averageCpuUsage == null && memoryUsage == null)
                    {
                        throw new ArgumentException(nameof(averageCpuUsage), nameof(memoryUsage));
                    }

                    if (averageCpuUsage != null && memoryUsage != null)
                    {
                        _log.Info(
                            $"High CPU usage ({averageCpuUsage}%) and high memory usage ({memoryUsage}%) detected. " +
                            $"Browser count decreased from {counter.createdHtmlRenderCount + 1} to {counter.createdHtmlRenderCount}."
                            );
                    }
                    else if (averageCpuUsage != null)
                    {
                        _log.Info(
                            $"High CPU usage ({averageCpuUsage}%) detected. " +
                            $"Browser count decreased from {counter.createdHtmlRenderCount + 1} to {counter.createdHtmlRenderCount}."
                            );
                    }
                    else
                    {
                        _log.Info(
                            $"High memory usage ({memoryUsage}%) detected. " +
                            $"Browser count decreased from {counter.createdHtmlRenderCount + 1} to {counter.createdHtmlRenderCount}."
                            );
                    }
                };
            }
        }