private static async Task SimpleDemoWithDelayAsync() { Console.WriteLine("TransformBlockDemo has started!"); var block = new TransformBlock <int, string>( // by default singlethreaded async(input) => { await Task.Delay(500).ConfigureAwait(false); return(input.ToString()); }); for (int i = 0; i < 10; i++) { block.Post(i); Console.WriteLine($"TransformBlock input queue count: {block.InputCount}"); } block.Complete(); // No mo data. while (await block.OutputAvailableAsync().ConfigureAwait(false)) { Console.WriteLine($"TransformBlock OutputCount: {block.InputCount}"); var output = await block.ReceiveAsync().ConfigureAwait(false); Console.WriteLine($"TransformBlock TransformOutput: {output}"); Console.WriteLine($"TransformBlock OutputCount: {block.OutputCount}"); // will always be 0, since receive data is a blocking action and this transformblock is single threaded } // wait for completion. await block.Completion.ConfigureAwait(false); Console.WriteLine("Finished!"); Console.ReadKey(); }
/// <summary>High throughput parallel lazy-ish method</summary> /// <remarks> /// Inspired by https://stackoverflow.com/a/58564740/1128762 /// </remarks> public static async IAsyncEnumerable <TResult> ParallelSelectAwait <TArg, TResult>(this IEnumerable <TArg> source, Func <TArg, Task <TResult> > selector, int maxDop, [EnumeratorCancellation] CancellationToken token = default) { var processor = new TransformBlock <TArg, TResult>(selector, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDop, BoundedCapacity = (maxDop * 5) / 4, CancellationToken = token }); foreach (var item in source) { while (!processor.Post(item)) { yield return(await ReceiveAsync()); } if (processor.TryReceive(out var result)) { yield return(result); } } processor.Complete(); while (await processor.OutputAvailableAsync(token)) { yield return(Receive()); } async Task <TResult> ReceiveAsync() { if (!await processor.OutputAvailableAsync() && !token.IsCancellationRequested) { throw new InvalidOperationException("No output available after posting output and waiting"); } return(Receive()); } TResult Receive() { token.ThrowIfCancellationRequested(); if (!processor.TryReceive(out var result)) { throw new InvalidOperationException("Nothing received even though output available"); } return(result); } }
public void SingleTaskExceptioned_AsyncFunc() { var block = new TransformBlock <int, string>(async i => { Console.WriteLine("Starting block with input: " + i); await Task.Delay(100).ConfigureAwait(false); if (i == 2) { throw new ArgumentException("i == 2", nameof(i)); } else { await Task.Delay(500 / (i * 2 + 1)).ConfigureAwait(false); } return($"{i} completed"); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1 }); for (int i = 0; i < 5; i++) { block.SendAsync(i).Wait(); } int received = 0; var outputs = new List <string>(); while (received < 5) { block.OutputAvailableAsync().Wait(); if (received == 2) { Assert.ThrowsException <ArgumentException>(() => block.TryReceive(out var output)); outputs.Add("2 Exception"); received++; } else { if (block.TryReceive(out var output)) { received++; if (received == 2) { Console.WriteLine("this should've errored"); } outputs.Add(output); } } } for (int i = 0; i < 5; i++) { if (i != 2) { Assert.IsTrue(outputs[i].Contains(i.ToString())); } } }
/// <summary> /// Simplified method for async operations that don't need to be chained, and when the result can fit in memory /// </summary> public static async Task <IReadOnlyCollection <R> > BlockTransform <T, R>(this IEnumerable <T> source, Func <T, Task <R> > transform, int parallelism = 1, int?capacity = null, Action <BulkProgressInfo <R> > progressUpdate = null, TimeSpan progressPeriod = default(TimeSpan)) { progressPeriod = progressPeriod == default(TimeSpan) ? 10.Seconds() : progressPeriod; var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = parallelism }; if (capacity.HasValue) { options.BoundedCapacity = capacity.Value; } var block = new TransformBlock <T, R>(transform, options); var totalProgress = Stopwatch.StartNew(); var swProgress = Stopwatch.StartNew(); // by producing asynchronously and using SendAsync we can throttle how much we can form the source and consume at the same time var produce = Produce(source, block); var result = new List <R>(); var newResults = new List <R>(); while (true) { var outputAvailableTask = block.OutputAvailableAsync(); var completedTask = await Task.WhenAny(outputAvailableTask, Task.Delay(progressPeriod)); if (completedTask == outputAvailableTask) { var available = await outputAvailableTask; if (!available) { break; } var item = await block.ReceiveAsync(); newResults.Add(item); result.Add(item); } var elapsed = swProgress.Elapsed; if (elapsed > progressPeriod) { progressUpdate?.Invoke(new BulkProgressInfo <R>(result, newResults, elapsed)); swProgress.Restart(); newResults.Clear(); } } progressUpdate?.Invoke(new BulkProgressInfo <R>(result, result, totalProgress.Elapsed)); await Task.WhenAll(produce, block.Completion); return(result); }
private static async Task <List <DispatchingResult> > ConsumeProjectionsFlow(TransformBlock <List <ProjectionDescriptor>, List <DispatchingResult> > flow, int activeDescriptors, CancellationToken token) { var results = new List <DispatchingResult>(activeDescriptors); while (await flow.OutputAvailableAsync(token).NotOnCapturedContext()) { var r = await flow.ReceiveAsync(token).NotOnCapturedContext(); results.AddRange(r); } return(results); }
public void SingleThreaded_WaitAvailable() { var numInputs = 10; var boundedCapacity = numInputs; var maxDegreeParallelism = 1; var block = new TransformBlock <BlockTestInput, BlockTestOutput>(async i => { var startTime = DateTime.Now; await Task.Delay(i.TaskDuration); return(new BlockTestOutput(i, startTime, DateTime.Now)); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = boundedCapacity, MaxDegreeOfParallelism = maxDegreeParallelism }); var inputList = new Dictionary <int, BlockTestInput>(); for (int i = 0; i < numInputs; i++) { inputList.Add(i, new BlockTestInput(i, new TimeSpan(0, 0, 0, 0, 1))); } foreach (var item in inputList) { block.SendAsync(item.Value); } block.Complete(); block.Completion.Wait(); var outputList = new List <BlockTestOutput>(); for (int i = 0; i < numInputs; i++) { block.OutputAvailableAsync().Wait(); Assert.IsTrue(block.TryReceive(out var output)); outputList.Add(output); } Assert.AreEqual(numInputs, outputList.Count); var lastId = -1; var lastStart = DateTime.MinValue; var lastCompletion = DateTime.MinValue; foreach (var output in outputList) { Assert.AreEqual(lastId + 1, output.Input.Id); Assert.IsTrue(lastStart < output.TaskStart); Assert.IsTrue(lastCompletion < output.TaskFinished); lastStart = output.TaskStart; lastCompletion = output.TaskFinished; lastId++; } }
public IEnumerable <ConsumerResult <TResultPayload> > Generate <TResultPayload>(ISourceCodeProvider dataProvider, IConsumer <TResultPayload> consumer) { if (dataProvider == null) { throw new ArgumentNullException("dataProvider"); } if (consumer == null) { throw new ArgumentNullException("consumer"); } var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; var processingTaskRestriction = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = _testsGeneratorRestrictions.MaxProcessingTasksCount }; var outputTaskRestriction = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = _testsGeneratorRestrictions.MaxWritingTasksCount }; var producerBuffer = new TransformBlock <string, TestClassInMemoryInfo>(new Func <string, TestClassInMemoryInfo>(Produce), processingTaskRestriction); var generatedTestsBuffer = new TransformBlock <TestClassInMemoryInfo, ConsumerResult <TResultPayload> >( new Func <TestClassInMemoryInfo, ConsumerResult <TResultPayload> >(consumer.Consume), outputTaskRestriction); producerBuffer.LinkTo(generatedTestsBuffer, linkOptions); _additionalProducerBuffer.LinkTo(generatedTestsBuffer, linkOptions); var consumerResults = Task.Run(async delegate { List <ConsumerResult <TResultPayload> > consumerResultsBuffer = new List <ConsumerResult <TResultPayload> >(); while (await generatedTestsBuffer.OutputAvailableAsync()) { consumerResultsBuffer.Add(generatedTestsBuffer.Receive()); } return(consumerResultsBuffer); }); Parallel.ForEach(dataProvider.Provide(), async dataInMemory => { await producerBuffer.SendAsync(dataInMemory); }); producerBuffer.Complete(); consumerResults.Wait(); return(consumerResults.Result); }
public async Task TestProducerConsumer() { foreach (TaskScheduler scheduler in new[] { TaskScheduler.Default, new ConcurrentExclusiveSchedulerPair().ConcurrentScheduler }) { foreach (int maxMessagesPerTask in new[] { DataflowBlockOptions.Unbounded, 1, 2 }) { foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 1, 2 }) { foreach (int dop in new[] { 1, 2 }) { foreach (bool sync in DataflowTestHelpers.BooleanValues) { const int Messages = 100; var options = new ExecutionDataflowBlockOptions { BoundedCapacity = boundedCapacity, MaxDegreeOfParallelism = dop, MaxMessagesPerTask = maxMessagesPerTask, TaskScheduler = scheduler }; TransformBlock <int, int> tb = sync ? new TransformBlock <int, int>(i => i, options) : new TransformBlock <int, int>(i => TaskShim.Run(() => i), options); await TaskShim.WhenAll( TaskShim.Run(async delegate { // consumer int i = 0; while (await tb.OutputAvailableAsync()) { Assert.Equal(expected: i, actual: await tb.ReceiveAsync()); i++; } }), TaskShim.Run(async delegate { // producer for (int i = 0; i < Messages; i++) { await tb.SendAsync(i); } tb.Complete(); })); } } } } } }
private async Task <List <DispatchingResult> > ConsumeProjectionDispatchersFlow(TransformBlock <DispatchingContext, DispatchingResult> flow, CancellationToken token) { var capacity = flow.InputCount + flow.OutputCount; if (capacity == 0) { capacity = BatchSize / 4; } var results = new List <DispatchingResult>(capacity); while (await flow.OutputAvailableAsync(token).NotOnCapturedContext()) { var e = await flow.ReceiveAsync(token).NotOnCapturedContext(); results.Add(e); } return(results); }
private async Task <List <MessageEnvelope> > ConsumeDeserializedEnvelopes(TransformBlock <MessageRaw, MessageEnvelope> deserializeBlock, CancellationToken token) { var capacity = deserializeBlock.InputCount + deserializeBlock.OutputCount; if (capacity == 0) { capacity = BatchSize / 4; } var envelopes = new List <MessageEnvelope>(capacity); while (await deserializeBlock.OutputAvailableAsync(token).NotOnCapturedContext()) { var e = await deserializeBlock.ReceiveAsync(token).NotOnCapturedContext(); envelopes.Add(e); } return(envelopes); }
public static async Task Run() { var tr2Block = new TransformBlock <int, int>( x => { var result = new Task <int>(() => { Thread.Sleep(1000); return(x + x + x); }); result.Start(); return(result); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); var tr3Block = new TransformBlock <int, string>( n => n.ToString(CultureInfo.InvariantCulture)); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; tr2Block.LinkTo(tr3Block, linkOptions); for (var i = 0; i < 10; i++) { tr2Block.Post(i); } tr2Block.Complete(); while (await tr3Block.OutputAvailableAsync()) { while (tr3Block.TryReceive(out var item)) { Console.WriteLine($"TPL2: {item}"); } } await tr3Block.Completion; }
private static async Task SimpleDemoWithParallelismAsync() { Console.WriteLine("TransformBlockDemo has started!"); var block = new TransformBlock <int, string>( async(input) => { await Task.Delay(500).ConfigureAwait(false); return(input.ToString()); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4 }); // how to make the same code above parallel with adjusting options instead of code for (int i = 0; i < 10; i++) { block.Post(i); Console.WriteLine($"TransformBlock input queue count: {block.InputCount}"); } block.Complete(); // No mo data. while (await block.OutputAvailableAsync().ConfigureAwait(false)) { Console.WriteLine($"TransformBlock InputCount: {block.InputCount}"); var output = await block.ReceiveAsync().ConfigureAwait(false); Console.WriteLine($"TransformBlock TransformOutput: {output}"); Console.WriteLine($"TransformBlock OutputCount: {block.OutputCount}"); } // wait for completion. await block.Completion.ConfigureAwait(false); Console.WriteLine("Finished!"); Console.ReadKey(); }
// TODO: Abort early when bsaber.com is down (check if all items in block failed?) // TODO: Make cancellationToken actually do something. /// <summary> /// Gets all songs from the feed defined by the provided settings. /// </summary> /// <param name="settings"></param> /// <param name="cancellationToken"></param> /// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception> /// <exception cref="ArgumentException">Thrown when trying to access a feed that requires a username and the username wasn't provided.</exception> /// <returns></returns> public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings settings, CancellationToken cancellationToken) { if (cancellationToken != CancellationToken.None) { Logger.Warning("CancellationToken in GetSongsFromFeedAsync isn't implemented."); } if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync."); } Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>(); if (!(settings is BeastSaberFeedSettings _settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim())) { Logger.Error($"Can't access feed without a valid username in the config file"); throw new ArgumentException("Cannot access this feed without a valid username."); } int pageIndex = settings.StartingPage; int maxPages = _settings.MaxPages; bool useMaxSongs = _settings.MaxSongs != 0; bool useMaxPages = maxPages != 0; if (useMaxPages && pageIndex > 1) { maxPages = maxPages + pageIndex - 1; } var ProcessPageBlock = new TransformBlock <Uri, List <ScrapedSong> >(async feedUrl => { Stopwatch sw = new Stopwatch(); sw.Start(); //Logger.Debug($"Checking URL: {feedUrl}"); string pageText = ""; ContentType contentType; string contentTypeStr = string.Empty; try { using (var response = await WebUtils.WebClient.GetAsync(feedUrl).ConfigureAwait(false)) { contentTypeStr = response.Content.ContentType.ToLower(); if (ContentDictionary.ContainsKey(contentTypeStr)) { contentType = ContentDictionary[contentTypeStr]; } else { contentType = ContentType.Unknown; } pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } } catch (HttpRequestException ex) { Logger.Exception($"Error downloading {feedUrl} in TransformBlock.", ex); return(new List <ScrapedSong>()); } var newSongs = GetSongsFromPageText(pageText, feedUrl, contentType); sw.Stop(); //Logger.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms"); return(newSongs.Count > 0 ? newSongs : null); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxConcurrency, BoundedCapacity = MaxConcurrency, EnsureOrdered = true }); bool continueLooping = true; int itemsInBlock = 0; do { while (continueLooping) { var feedUrl = GetPageUri(Feeds[_settings.Feed].BaseUrl, pageIndex); await ProcessPageBlock.SendAsync(feedUrl).ConfigureAwait(false); // TODO: Need check with SongsPerPage itemsInBlock++; pageIndex++; if (pageIndex > maxPages && useMaxPages) { continueLooping = false; } while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping) { if (itemsInBlock <= 0) { break; } await ProcessPageBlock.OutputAvailableAsync().ConfigureAwait(false); while (ProcessPageBlock.TryReceive(out List <ScrapedSong> newSongs)) { itemsInBlock--; if (newSongs == null) { Logger.Debug("Received no new songs, last page reached."); ProcessPageBlock.Complete(); itemsInBlock = 0; continueLooping = false; break; } Logger.Debug($"Receiving {newSongs.Count} potential songs from {newSongs.First().SourceUri}"); foreach (var song in newSongs) { if (retDict.ContainsKey(song.Hash)) { Logger.Debug($"Song {song.Hash} already exists."); } else { if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0) { retDict.Add(song.Hash, song); } if (retDict.Count >= settings.MaxSongs && useMaxSongs) { continueLooping = false; } } } if (!useMaxPages || pageIndex <= maxPages) { if (retDict.Count < settings.MaxSongs) { continueLooping = true; } } } } } }while (continueLooping); return(retDict); }
/// <summary>High throughput parallel lazy-ish method</summary> /// <remarks> /// Inspired by https://stackoverflow.com/a/58564740/1128762 /// </remarks> public static async IAsyncEnumerable <TResult> ParallelSelectAwait <TArg, TResult>(this IEnumerable <TArg> source, Func <TArg, Task <TResult> > selector, int maxDop, [EnumeratorCancellation] CancellationToken token = default) { var processor = new TransformBlock <TArg, TResult>(selector, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDop, BoundedCapacity = (maxDop * 5) / 4, CancellationToken = token, SingleProducerConstrained = true, EnsureOrdered = false }); bool pipelineTerminatedEarly = false; foreach (var item in source) { while (!processor.Post(item)) { var result = await ReceiveAsync(); if (pipelineTerminatedEarly) { break; } yield return(result); } if (pipelineTerminatedEarly) { break; } if (processor.TryReceive(out var resultIfAvailable)) { yield return(resultIfAvailable); } } processor.Complete(); while (await processor.OutputAvailableAsync(token)) { var result = ReceiveKnownAvailable(); if (pipelineTerminatedEarly) { break; } yield return(result); } await processor.Completion; if (pipelineTerminatedEarly) { throw new InvalidOperationException("Pipeline terminated early missing items, but no exception thrown"); } async Task <TResult> ReceiveAsync() { await processor.OutputAvailableAsync(); return(ReceiveKnownAvailable()); } TResult ReceiveKnownAvailable() { token.ThrowIfCancellationRequested(); if (!processor.TryReceive(out var item)) { pipelineTerminatedEarly = true; return(default);
// TODO: Abort early when bsaber.com is down (check if all items in block failed?) // TODO: Make cancellationToken actually do something. /// <summary> /// Gets all songs from the feed defined by the provided settings. /// </summary> /// <param name="settings"></param> /// <param name="cancellationToken"></param> /// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception> /// <exception cref="ArgumentException">Thrown when trying to access a feed that requires a username and the username wasn't provided.</exception> /// <exception cref="OperationCanceledException"></exception> /// <returns></returns> public async Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return(FeedResult.CancelledResult); } if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync."); } Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>(); if (!(settings is BeastSaberFeedSettings _settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim())) { //Logger?.Error($"Can't access feed without a valid username in the config file"); throw new ArgumentException("Cannot access this feed without a valid username."); } int pageIndex = settings.StartingPage; int maxPages = _settings.MaxPages; bool useMaxSongs = _settings.MaxSongs != 0; bool useMaxPages = maxPages != 0; if (useMaxPages && pageIndex > 1) { maxPages = maxPages + pageIndex - 1; } var ProcessPageBlock = new TransformBlock <Uri, PageReadResult>(async feedUri => { Stopwatch sw = new Stopwatch(); sw.Start(); //Logger?.Debug($"Checking URL: {feedUrl}"); string pageText = ""; ContentType contentType = ContentType.Unknown; string contentTypeStr = string.Empty; IWebResponseMessage response = null; try { response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false); if ((response?.StatusCode ?? 500) == 500) { response?.Dispose(); response = null; Logger?.Warning($"Internal server error on {feedUri}, retrying in 20 seconds"); await Task.Delay(20000).ConfigureAwait(false); response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false); } response.EnsureSuccessStatusCode(); contentTypeStr = response.Content.ContentType.ToLower(); if (ContentDictionary.ContainsKey(contentTypeStr)) { contentType = ContentDictionary[contentTypeStr]; } else { contentType = ContentType.Unknown; } pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); } catch (WebClientException ex) { return(PageReadResult.FromWebClientException(ex, feedUri)); } catch (OperationCanceledException) { return(new PageReadResult(feedUri, null, new FeedReaderException("Page read was cancelled.", new OperationCanceledException(), FeedReaderFailureCode.Cancelled), PageErrorType.Cancelled)); } catch (Exception ex) { string message = $"Error downloading {feedUri} in TransformBlock."; Logger?.Debug(message); Logger?.Debug($"{ex.Message}\n{ex.StackTrace}"); return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown)); } finally { response?.Dispose(); response = null; } List <ScrapedSong> newSongs = null; try { newSongs = GetSongsFromPageText(pageText, feedUri, contentType); } catch (JsonReaderException ex) { // TODO: Probably don't need a logger message here, caller can deal with it. string message = $"Error parsing page text for {feedUri} in TransformBlock."; Logger?.Debug(message); Logger?.Debug($"{ex.Message}\n{ex.StackTrace}"); return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError)); } catch (XmlException ex) { // TODO: Probably don't need a logger message here, caller can deal with it. string message = $"Error parsing page text for {feedUri} in TransformBlock."; Logger?.Debug(message); Logger?.Debug($"{ex.Message}\n{ex.StackTrace}"); return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError)); } catch (Exception ex) { // TODO: Probably don't need a logger message here, caller can deal with it. string message = $"Uncaught error parsing page text for {feedUri} in TransformBlock."; Logger?.Debug(message); Logger?.Debug($"{ex.Message}\n{ex.StackTrace}"); return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown)); } sw.Stop(); //Logger?.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms"); return(new PageReadResult(feedUri, newSongs)); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxConcurrency, BoundedCapacity = MaxConcurrency, CancellationToken = cancellationToken //#if NETSTANDARD // , EnsureOrdered = true //#endif }); bool continueLooping = true; int itemsInBlock = 0; List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2); do { if (cancellationToken.IsCancellationRequested) { continueLooping = false; } while (continueLooping) { if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false); } if (cancellationToken.IsCancellationRequested) { continueLooping = false; break; } var feedUrl = GetPageUri(Feeds[_settings.Feed].BaseUrl, pageIndex); await ProcessPageBlock.SendAsync(feedUrl, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage itemsInBlock++; pageIndex++; if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested) { continueLooping = false; } // TODO: Better http error handling, what if only a single page is broken and returns 0 songs? while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping) { if (cancellationToken.IsCancellationRequested) { continueLooping = false; break; } if (itemsInBlock <= 0) { break; } await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false); while (ProcessPageBlock.TryReceive(out PageReadResult pageResult)) { if (pageResult != null) { pageResults.Add(pageResult); } if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false); } itemsInBlock--; if (pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error. { Logger?.Debug("Received no new songs, last page reached."); ProcessPageBlock.Complete(); itemsInBlock = 0; continueLooping = false; break; } if (pageResult.Count > 0) { Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}"); } else { Logger?.Debug($"Did not find any songs in {Name}.{settings.FeedName}."); } // TODO: Process PageReadResults for better error feedback. foreach (var song in pageResult.Songs) { if (!retDict.ContainsKey(song.Hash)) { if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0) { retDict.Add(song.Hash, song); } if (retDict.Count >= settings.MaxSongs && useMaxSongs) { continueLooping = false; } } } if (!useMaxPages || pageIndex <= maxPages) { if (retDict.Count < settings.MaxSongs) { continueLooping = true; } } } } } }while (continueLooping); return(new FeedResult(retDict, pageResults)); }
// TODO: Abort early when bsaber.com is down (check if all items in block failed?) // TODO: Make cancellationToken actually do something. /// <summary> /// Gets all songs from the feed defined by the provided settings. /// </summary> /// <param name="settings"></param> /// <param name="cancellationToken"></param> /// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception> /// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception> /// <exception cref="OperationCanceledException"></exception> /// <returns></returns> public async override Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return(FeedResult.CancelledResult); } if (settings == null) { throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync."); } Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>(); if (!(settings is BeastSaberFeedSettings _settings)) { throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE); } if (_settings.Feed != BeastSaberFeedName.CuratorRecommended && string.IsNullOrEmpty(_settings.Username)) { _settings.Username = Username; } BeastSaberFeed feed = new BeastSaberFeed(_settings) { StoreRawData = StoreRawData }; try { feed.EnsureValidSettings(); } catch (InvalidFeedSettingsException ex) { return(new FeedResult(null, null, ex, FeedResultError.Error)); } int pageIndex = settings.StartingPage; int maxPages = _settings.MaxPages; int pagesChecked = 0; bool useMaxSongs = _settings.MaxSongs != 0; bool useMaxPages = maxPages != 0; if (useMaxPages && pageIndex > 1) { maxPages = maxPages + pageIndex - 1; } var ProcessPageBlock = new TransformBlock <int, PageReadResult>(async pageNum => { return(await feed.GetSongsFromPageAsync(pageNum, cancellationToken).ConfigureAwait(false)); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxConcurrency, BoundedCapacity = MaxConcurrency, CancellationToken = cancellationToken //#if NETSTANDARD // , EnsureOrdered = true //#endif }); bool continueLooping = true; int itemsInBlock = 0; List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2); do { if (cancellationToken.IsCancellationRequested) { continueLooping = false; } while (continueLooping) { if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false); } if (cancellationToken.IsCancellationRequested) { continueLooping = false; break; } await ProcessPageBlock.SendAsync(pageIndex, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage itemsInBlock++; pageIndex++; if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested) { continueLooping = false; } // TODO: Better http error handling, what if only a single page is broken and returns 0 songs? while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping) { if (cancellationToken.IsCancellationRequested) { continueLooping = false; break; } if (itemsInBlock <= 0) { break; } await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false); while (ProcessPageBlock.TryReceive(out PageReadResult pageResult)) { int songsAdded = 0; if (pageResult != null) { pageResults.Add(pageResult); } if (Utilities.IsPaused) { await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false); } itemsInBlock--; if (pageResult.IsLastPage || pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error. { Logger?.Debug("Received no new songs, last page reached."); ProcessPageBlock.Complete(); itemsInBlock = 0; continueLooping = false; break; } if (pageResult.Count > 0) { Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}"); } else { Logger?.Debug($"Did not find any songs on page '{pageResult.Uri}' of {Name}.{settings.FeedName}."); } // TODO: Process PageReadResults for better error feedback. foreach (var song in pageResult.Songs) { if (!retDict.ContainsKey(song.Hash)) { if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0) { retDict.Add(song.Hash, song); songsAdded++; } if (retDict.Count >= settings.MaxSongs && useMaxSongs) { continueLooping = false; } } } int prog = Interlocked.Increment(ref pagesChecked); progress?.Report(new ReaderProgress(prog, songsAdded)); if (!useMaxPages || pageIndex <= maxPages) { if (retDict.Count < settings.MaxSongs) { continueLooping = true; } } } } } }while (continueLooping); if (pageResults.Any(r => r.PageError == PageErrorType.Cancelled)) { return(FeedResult.GetCancelledResult(retDict, pageResults)); } return(new FeedResult(retDict, pageResults)); }
public static async Task <IReadOnlyCollection <GraphTaskResult> > Run(this TaskGraph tasks, int parallel, ILogger log, CancellationToken cancel) { async Task <GraphTaskResult> RunTask(GraphTask task) { var sw = Stopwatch.StartNew(); GraphTaskResult Result(Exception ex = null) => new GraphTaskResult { Name = task.Name, FinalStatus = task.Status, Duration = sw.Elapsed, Exception = ex }; try { if (cancel.IsCancellationRequested || tasks.DependenciesDeep(task).Any(d => d.Status.In(Cancelled, Error))) { task.Status = Cancelled; return(Result()); } task.Status = Running; log = log.ForContext("Task", task.Name); await task.Run(log, cancel); if (cancel.IsCancellationRequested) { task.Status = Cancelled; } else { task.Status = Success; } return(Result()); } catch (Exception ex) { task.Status = Error; log.Error(ex, "Task {Task} failed: {Message}", task.Name, ex.Message); return(Result(ex)); } } var block = new TransformBlock <GraphTask, GraphTaskResult>(RunTask, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = parallel }); var newTaskSignal = new AsyncManualResetEvent(true); async Task Producer() { while (!tasks.AllComplete) { if (cancel.IsCancellationRequested) { foreach (var t in tasks.All.Where(t => t.Status.IsIncomplete())) { t.Status = Cancelled; } } var tasksToAdd = tasks.AvailableToRun().ToList(); if (tasksToAdd.IsEmpty()) { // if no tasks are ready to start. Wait to either be signaled, or log which tasks are still running var logTimeTask = Task.Delay(1.Minutes(), cancel); await Task.WhenAny(logTimeTask, newTaskSignal.WaitAsync()); if (newTaskSignal.IsSet) { newTaskSignal.Reset(); } if (logTimeTask.IsCompleted) { log.Debug("Waiting for {TaskList} to complete", tasks.Running.Select(t => t.Name)); } } foreach (var task in tasksToAdd) { task.Status = Queued; await block.SendAsync(task); } } block.Complete(); } var producer = Producer(); var taskResults = new List <GraphTaskResult>(); while (await block.OutputAvailableAsync()) { var item = await block.ReceiveAsync(); taskResults.Add(item); newTaskSignal.Set(); } await Task.WhenAll(producer, block.Completion); return(taskResults); }