Beispiel #1
0
        public OutputtersFlow CreateFlow(string storageName)
        {
            if (string.IsNullOrWhiteSpace(storageName))
            {
                storageName = _defaultStorageName;

                _logger.Info("Storage name is empty, using the default value.");
            }

            var outputtersFunc = _outputtersAsync.Select(outputterAsync =>
            {
                return(new Action <RatingDataContainer>(result => Console.WriteLine("Got result.")));
            });

            var outputtersFlow = new OutputtersFlow(outputtersFunc.Take(1));

            _logger.Info("Constructed outputters pipeline.");
            return(outputtersFlow);
        }
Beispiel #2
0
        private DataflowPipeline ConstructPipeline(string storageName)
        {
            // Input component work.
            InputtersFlow inputtersFlow = InputManagerAsync.CreateFlow(storageName);

            // Crawlers component work.
            CrawlersFlow crawlersFlow = CrawlersManagerAsync.CreateFlow();

            // Appraisers component work.
            AppraisersFlow appraisersFlow = AppraisersManagerAsync.CreateFlow();

            // Output component work.
            OutputtersFlow outputtersFlow =
                OutputManagerAsync.CreateFlow(storageName: string.Empty);

            // Constructing pipeline.
            inputtersFlow.LinkTo(crawlersFlow);
            crawlersFlow.LinkTo(appraisersFlow);
            appraisersFlow.LinkTo(outputtersFlow);

            return(new DataflowPipeline(inputtersFlow, outputtersFlow));
        }
Beispiel #3
0
        public async Task <bool> SaveResults(OutputtersFlow outputtersFlow, string storageName)
        {
            if (string.IsNullOrWhiteSpace(storageName))
            {
                storageName = _defaultStorageName;

                const string message = "Storage name is empty, using the default value.";
                _logger.Info(message);
                GlobalMessageHandler.OutputMessage(message);
            }

            // Make sure that the final pipeline task is completed.
            await outputtersFlow.CompletionTask;

            IReadOnlyList <RatingDataContainer> results = outputtersFlow.Results.ToReadOnlyList();

            IReadOnlyList <List <RatingDataContainer> > resultsToSave = results
                                                                        .GroupBy(rating => rating.RatingId, (key, group) => group.ToList())
                                                                        .ToReadOnlyList();

            resultsToSave
            .AsParallel()
            .ForAll(rating => rating.Sort((x, y) => y.RatingValue.CompareTo(x.RatingValue)));

            IReadOnlyList <Task <bool> > resultTasks = _outputtersAsync
                                                       .Select(outputterAsync => TryGetRatings(outputterAsync, resultsToSave, storageName))
                                                       .ToReadOnlyList();

            IReadOnlyList <bool> statuses = await Task.WhenAll(resultTasks);

            if (statuses.Count > 0 && statuses.All(statis => statis))
            {
                _logger.Info($"Successfully saved all results to \"{storageName}\".");
                return(true);
            }

            _logger.Info($"Could not save some results to \"{storageName}\".");
            return(false);
        }
Beispiel #4
0
        private async Task <ServiceStatus> SaveResults(OutputtersFlow outputtersFlow)
        {
            try
            {
                bool status = await OutputManagerAsync.SaveResults(
                    outputtersFlow, storageName : string.Empty
                    );

                if (status)
                {
                    GlobalMessageHandler.OutputMessage("Ratings was saved successfully.");
                    return(ServiceStatus.Ok);
                }

                GlobalMessageHandler.OutputMessage("Ratings was not saved.");
                return(ServiceStatus.OutputUnsaved);
            }
            catch (Exception ex)
            {
                _logger.Error(ex, "Exception occured during saving results work.");
                return(ServiceStatus.OutputError);
            }
        }