public OutputtersFlow CreateFlow(string storageName) { if (string.IsNullOrWhiteSpace(storageName)) { storageName = _defaultStorageName; _logger.Info("Storage name is empty, using the default value."); } var outputtersFunc = _outputtersAsync.Select(outputterAsync => { return(new Action <RatingDataContainer>(result => Console.WriteLine("Got result."))); }); var outputtersFlow = new OutputtersFlow(outputtersFunc.Take(1)); _logger.Info("Constructed outputters pipeline."); return(outputtersFlow); }
private DataflowPipeline ConstructPipeline(string storageName) { // Input component work. InputtersFlow inputtersFlow = InputManagerAsync.CreateFlow(storageName); // Crawlers component work. CrawlersFlow crawlersFlow = CrawlersManagerAsync.CreateFlow(); // Appraisers component work. AppraisersFlow appraisersFlow = AppraisersManagerAsync.CreateFlow(); // Output component work. OutputtersFlow outputtersFlow = OutputManagerAsync.CreateFlow(storageName: string.Empty); // Constructing pipeline. inputtersFlow.LinkTo(crawlersFlow); crawlersFlow.LinkTo(appraisersFlow); appraisersFlow.LinkTo(outputtersFlow); return(new DataflowPipeline(inputtersFlow, outputtersFlow)); }
public async Task <bool> SaveResults(OutputtersFlow outputtersFlow, string storageName) { if (string.IsNullOrWhiteSpace(storageName)) { storageName = _defaultStorageName; const string message = "Storage name is empty, using the default value."; _logger.Info(message); GlobalMessageHandler.OutputMessage(message); } // Make sure that the final pipeline task is completed. await outputtersFlow.CompletionTask; IReadOnlyList <RatingDataContainer> results = outputtersFlow.Results.ToReadOnlyList(); IReadOnlyList <List <RatingDataContainer> > resultsToSave = results .GroupBy(rating => rating.RatingId, (key, group) => group.ToList()) .ToReadOnlyList(); resultsToSave .AsParallel() .ForAll(rating => rating.Sort((x, y) => y.RatingValue.CompareTo(x.RatingValue))); IReadOnlyList <Task <bool> > resultTasks = _outputtersAsync .Select(outputterAsync => TryGetRatings(outputterAsync, resultsToSave, storageName)) .ToReadOnlyList(); IReadOnlyList <bool> statuses = await Task.WhenAll(resultTasks); if (statuses.Count > 0 && statuses.All(statis => statis)) { _logger.Info($"Successfully saved all results to \"{storageName}\"."); return(true); } _logger.Info($"Could not save some results to \"{storageName}\"."); return(false); }
private async Task <ServiceStatus> SaveResults(OutputtersFlow outputtersFlow) { try { bool status = await OutputManagerAsync.SaveResults( outputtersFlow, storageName : string.Empty ); if (status) { GlobalMessageHandler.OutputMessage("Ratings was saved successfully."); return(ServiceStatus.Ok); } GlobalMessageHandler.OutputMessage("Ratings was not saved."); return(ServiceStatus.OutputUnsaved); } catch (Exception ex) { _logger.Error(ex, "Exception occured during saving results work."); return(ServiceStatus.OutputError); } }