private static async Task SimpleDemoWithDelayAsync()
{
Console.WriteLine("TransformBlockDemo has started!");
var block = new TransformBlock <int, string>( // by default singlethreaded
async(input) =>
{
await Task.Delay(500).ConfigureAwait(false);
return(input.ToString());
});
for (int i = 0; i < 10; i++)
{
block.Post(i);
Console.WriteLine($"TransformBlock input queue count: {block.InputCount}");
}
block.Complete(); // No mo data.
while (await block.OutputAvailableAsync().ConfigureAwait(false))
{
Console.WriteLine($"TransformBlock OutputCount: {block.InputCount}");
var output = await block.ReceiveAsync().ConfigureAwait(false);
Console.WriteLine($"TransformBlock TransformOutput: {output}");
Console.WriteLine($"TransformBlock OutputCount: {block.OutputCount}"); // will always be 0, since receive data is a blocking action and this transformblock is single threaded
}
// wait for completion.
await block.Completion.ConfigureAwait(false);
Console.WriteLine("Finished!");
Console.ReadKey();
}
/// <summary>High throughput parallel lazy-ish method</summary>
/// <remarks>
/// Inspired by https://stackoverflow.com/a/58564740/1128762
/// </remarks>
public static async IAsyncEnumerable <TResult> ParallelSelectAwait <TArg, TResult>(this IEnumerable <TArg> source,
Func <TArg, Task <TResult> > selector, int maxDop, [EnumeratorCancellation] CancellationToken token = default)
{
var processor = new TransformBlock <TArg, TResult>(selector, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDop,
BoundedCapacity = (maxDop * 5) / 4,
CancellationToken = token
});
foreach (var item in source)
{
while (!processor.Post(item))
{
yield return(await ReceiveAsync());
}
if (processor.TryReceive(out var result))
{
yield return(result);
}
}
processor.Complete();
while (await processor.OutputAvailableAsync(token))
{
yield return(Receive());
}
async Task <TResult> ReceiveAsync()
{
if (!await processor.OutputAvailableAsync() && !token.IsCancellationRequested)
{
throw new InvalidOperationException("No output available after posting output and waiting");
}
return(Receive());
}
TResult Receive()
{
token.ThrowIfCancellationRequested();
if (!processor.TryReceive(out var result))
{
throw new InvalidOperationException("Nothing received even though output available");
}
return(result);
}
}
public void SingleTaskExceptioned_AsyncFunc()
{
var block = new TransformBlock <int, string>(async i =>
{
Console.WriteLine("Starting block with input: " + i);
await Task.Delay(100).ConfigureAwait(false);
if (i == 2)
{
throw new ArgumentException("i == 2", nameof(i));
}
else
{
await Task.Delay(500 / (i * 2 + 1)).ConfigureAwait(false);
}
return($"{i} completed");
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 10,
MaxDegreeOfParallelism = 1
});
for (int i = 0; i < 5; i++)
{
block.SendAsync(i).Wait();
}
int received = 0;
var outputs = new List <string>();
while (received < 5)
{
block.OutputAvailableAsync().Wait();
if (received == 2)
{
Assert.ThrowsException <ArgumentException>(() => block.TryReceive(out var output));
outputs.Add("2 Exception");
received++;
}
else
{
if (block.TryReceive(out var output))
{
received++;
if (received == 2)
{
Console.WriteLine("this should've errored");
}
outputs.Add(output);
}
}
}
for (int i = 0; i < 5; i++)
{
if (i != 2)
{
Assert.IsTrue(outputs[i].Contains(i.ToString()));
}
}
}
/// <summary>
/// Simplified method for async operations that don't need to be chained, and when the result can fit in memory
/// </summary>
public static async Task <IReadOnlyCollection <R> > BlockTransform <T, R>(this IEnumerable <T> source,
Func <T, Task <R> > transform, int parallelism = 1, int?capacity = null,
Action <BulkProgressInfo <R> > progressUpdate = null, TimeSpan progressPeriod = default(TimeSpan))
{
progressPeriod = progressPeriod == default(TimeSpan) ? 10.Seconds() : progressPeriod;
var options = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = parallelism
};
if (capacity.HasValue)
{
options.BoundedCapacity = capacity.Value;
}
var block = new TransformBlock <T, R>(transform, options);
var totalProgress = Stopwatch.StartNew();
var swProgress = Stopwatch.StartNew();
// by producing asynchronously and using SendAsync we can throttle how much we can form the source and consume at the same time
var produce = Produce(source, block);
var result = new List <R>();
var newResults = new List <R>();
while (true)
{
var outputAvailableTask = block.OutputAvailableAsync();
var completedTask = await Task.WhenAny(outputAvailableTask, Task.Delay(progressPeriod));
if (completedTask == outputAvailableTask)
{
var available = await outputAvailableTask;
if (!available)
{
break;
}
var item = await block.ReceiveAsync();
newResults.Add(item);
result.Add(item);
}
var elapsed = swProgress.Elapsed;
if (elapsed > progressPeriod)
{
progressUpdate?.Invoke(new BulkProgressInfo <R>(result, newResults, elapsed));
swProgress.Restart();
newResults.Clear();
}
}
progressUpdate?.Invoke(new BulkProgressInfo <R>(result, result, totalProgress.Elapsed));
await Task.WhenAll(produce, block.Completion);
return(result);
}
private static async Task <List <DispatchingResult> > ConsumeProjectionsFlow(TransformBlock <List <ProjectionDescriptor>, List <DispatchingResult> > flow, int activeDescriptors, CancellationToken token)
{
var results = new List <DispatchingResult>(activeDescriptors);
while (await flow.OutputAvailableAsync(token).NotOnCapturedContext())
{
var r = await flow.ReceiveAsync(token).NotOnCapturedContext();
results.AddRange(r);
}
return(results);
}
public void SingleThreaded_WaitAvailable()
{
var numInputs = 10;
var boundedCapacity = numInputs;
var maxDegreeParallelism = 1;
var block = new TransformBlock <BlockTestInput, BlockTestOutput>(async i =>
{
var startTime = DateTime.Now;
await Task.Delay(i.TaskDuration);
return(new BlockTestOutput(i, startTime, DateTime.Now));
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = boundedCapacity,
MaxDegreeOfParallelism = maxDegreeParallelism
});
var inputList = new Dictionary <int, BlockTestInput>();
for (int i = 0; i < numInputs; i++)
{
inputList.Add(i, new BlockTestInput(i, new TimeSpan(0, 0, 0, 0, 1)));
}
foreach (var item in inputList)
{
block.SendAsync(item.Value);
}
block.Complete();
block.Completion.Wait();
var outputList = new List <BlockTestOutput>();
for (int i = 0; i < numInputs; i++)
{
block.OutputAvailableAsync().Wait();
Assert.IsTrue(block.TryReceive(out var output));
outputList.Add(output);
}
Assert.AreEqual(numInputs, outputList.Count);
var lastId = -1;
var lastStart = DateTime.MinValue;
var lastCompletion = DateTime.MinValue;
foreach (var output in outputList)
{
Assert.AreEqual(lastId + 1, output.Input.Id);
Assert.IsTrue(lastStart < output.TaskStart);
Assert.IsTrue(lastCompletion < output.TaskFinished);
lastStart = output.TaskStart;
lastCompletion = output.TaskFinished;
lastId++;
}
}
public IEnumerable <ConsumerResult <TResultPayload> > Generate <TResultPayload>(ISourceCodeProvider dataProvider, IConsumer <TResultPayload> consumer)
{
if (dataProvider == null)
{
throw new ArgumentNullException("dataProvider");
}
if (consumer == null)
{
throw new ArgumentNullException("consumer");
}
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
var processingTaskRestriction = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _testsGeneratorRestrictions.MaxProcessingTasksCount
};
var outputTaskRestriction = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _testsGeneratorRestrictions.MaxWritingTasksCount
};
var producerBuffer = new TransformBlock <string, TestClassInMemoryInfo>(new Func <string, TestClassInMemoryInfo>(Produce), processingTaskRestriction);
var generatedTestsBuffer = new TransformBlock <TestClassInMemoryInfo, ConsumerResult <TResultPayload> >(
new Func <TestClassInMemoryInfo, ConsumerResult <TResultPayload> >(consumer.Consume), outputTaskRestriction);
producerBuffer.LinkTo(generatedTestsBuffer, linkOptions);
_additionalProducerBuffer.LinkTo(generatedTestsBuffer, linkOptions);
var consumerResults = Task.Run(async delegate {
List <ConsumerResult <TResultPayload> > consumerResultsBuffer = new List <ConsumerResult <TResultPayload> >();
while (await generatedTestsBuffer.OutputAvailableAsync())
{
consumerResultsBuffer.Add(generatedTestsBuffer.Receive());
}
return(consumerResultsBuffer);
});
Parallel.ForEach(dataProvider.Provide(), async dataInMemory => {
await producerBuffer.SendAsync(dataInMemory);
});
producerBuffer.Complete();
consumerResults.Wait();
return(consumerResults.Result);
}
public async Task TestProducerConsumer()
{
foreach (TaskScheduler scheduler in new[] { TaskScheduler.Default, new ConcurrentExclusiveSchedulerPair().ConcurrentScheduler })
{
foreach (int maxMessagesPerTask in new[] { DataflowBlockOptions.Unbounded, 1, 2 })
{
foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 1, 2 })
{
foreach (int dop in new[] { 1, 2 })
{
foreach (bool sync in DataflowTestHelpers.BooleanValues)
{
const int Messages = 100;
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = boundedCapacity,
MaxDegreeOfParallelism = dop,
MaxMessagesPerTask = maxMessagesPerTask,
TaskScheduler = scheduler
};
TransformBlock <int, int> tb = sync ?
new TransformBlock <int, int>(i => i, options) :
new TransformBlock <int, int>(i => TaskShim.Run(() => i), options);
await TaskShim.WhenAll(
TaskShim.Run(async delegate
{ // consumer
int i = 0;
while (await tb.OutputAvailableAsync())
{
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
i++;
}
}),
TaskShim.Run(async delegate
{ // producer
for (int i = 0; i < Messages; i++)
{
await tb.SendAsync(i);
}
tb.Complete();
}));
}
}
}
}
}
}
private async Task <List <DispatchingResult> > ConsumeProjectionDispatchersFlow(TransformBlock <DispatchingContext, DispatchingResult> flow, CancellationToken token)
{
var capacity = flow.InputCount + flow.OutputCount;
if (capacity == 0)
{
capacity = BatchSize / 4;
}
var results = new List <DispatchingResult>(capacity);
while (await flow.OutputAvailableAsync(token).NotOnCapturedContext())
{
var e = await flow.ReceiveAsync(token).NotOnCapturedContext();
results.Add(e);
}
return(results);
}
private async Task <List <MessageEnvelope> > ConsumeDeserializedEnvelopes(TransformBlock <MessageRaw, MessageEnvelope> deserializeBlock, CancellationToken token)
{
var capacity = deserializeBlock.InputCount + deserializeBlock.OutputCount;
if (capacity == 0)
{
capacity = BatchSize / 4;
}
var envelopes = new List <MessageEnvelope>(capacity);
while (await deserializeBlock.OutputAvailableAsync(token).NotOnCapturedContext())
{
var e = await deserializeBlock.ReceiveAsync(token).NotOnCapturedContext();
envelopes.Add(e);
}
return(envelopes);
}
public static async Task Run()
{
var tr2Block = new TransformBlock <int, int>(
x =>
{
var result = new Task <int>(() =>
{
Thread.Sleep(1000);
return(x + x + x);
});
result.Start();
return(result);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 2
});
var tr3Block = new TransformBlock <int, string>(
n => n.ToString(CultureInfo.InvariantCulture));
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
tr2Block.LinkTo(tr3Block, linkOptions);
for (var i = 0; i < 10; i++)
{
tr2Block.Post(i);
}
tr2Block.Complete();
while (await tr3Block.OutputAvailableAsync())
{
while (tr3Block.TryReceive(out var item))
{
Console.WriteLine($"TPL2: {item}");
}
}
await tr3Block.Completion;
}
private static async Task SimpleDemoWithParallelismAsync()
{
Console.WriteLine("TransformBlockDemo has started!");
var block = new TransformBlock <int, string>(
async(input) =>
{
await Task.Delay(500).ConfigureAwait(false);
return(input.ToString());
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4
}); // how to make the same code above parallel with adjusting options instead of code
for (int i = 0; i < 10; i++)
{
block.Post(i);
Console.WriteLine($"TransformBlock input queue count: {block.InputCount}");
}
block.Complete(); // No mo data.
while (await block.OutputAvailableAsync().ConfigureAwait(false))
{
Console.WriteLine($"TransformBlock InputCount: {block.InputCount}");
var output = await block.ReceiveAsync().ConfigureAwait(false);
Console.WriteLine($"TransformBlock TransformOutput: {output}");
Console.WriteLine($"TransformBlock OutputCount: {block.OutputCount}");
}
// wait for completion.
await block.Completion.ConfigureAwait(false);
Console.WriteLine("Finished!");
Console.ReadKey();
}
// TODO: Abort early when bsaber.com is down (check if all items in block failed?)
// TODO: Make cancellationToken actually do something.
/// <summary>
/// Gets all songs from the feed defined by the provided settings.
/// </summary>
/// <param name="settings"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception>
/// <exception cref="ArgumentException">Thrown when trying to access a feed that requires a username and the username wasn't provided.</exception>
/// <returns></returns>
public async Task <Dictionary <string, ScrapedSong> > GetSongsFromFeedAsync(IFeedSettings settings, CancellationToken cancellationToken)
{
if (cancellationToken != CancellationToken.None)
{
Logger.Warning("CancellationToken in GetSongsFromFeedAsync isn't implemented.");
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync.");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim()))
{
Logger.Error($"Can't access feed without a valid username in the config file");
throw new ArgumentException("Cannot access this feed without a valid username.");
}
int pageIndex = settings.StartingPage;
int maxPages = _settings.MaxPages;
bool useMaxSongs = _settings.MaxSongs != 0;
bool useMaxPages = maxPages != 0;
if (useMaxPages && pageIndex > 1)
{
maxPages = maxPages + pageIndex - 1;
}
var ProcessPageBlock = new TransformBlock <Uri, List <ScrapedSong> >(async feedUrl =>
{
Stopwatch sw = new Stopwatch();
sw.Start();
//Logger.Debug($"Checking URL: {feedUrl}");
string pageText = "";
ContentType contentType;
string contentTypeStr = string.Empty;
try
{
using (var response = await WebUtils.WebClient.GetAsync(feedUrl).ConfigureAwait(false))
{
contentTypeStr = response.Content.ContentType.ToLower();
if (ContentDictionary.ContainsKey(contentTypeStr))
{
contentType = ContentDictionary[contentTypeStr];
}
else
{
contentType = ContentType.Unknown;
}
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
}
catch (HttpRequestException ex)
{
Logger.Exception($"Error downloading {feedUrl} in TransformBlock.", ex);
return(new List <ScrapedSong>());
}
var newSongs = GetSongsFromPageText(pageText, feedUrl, contentType);
sw.Stop();
//Logger.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms");
return(newSongs.Count > 0 ? newSongs : null);
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxConcurrency,
BoundedCapacity = MaxConcurrency,
EnsureOrdered = true
});
bool continueLooping = true;
int itemsInBlock = 0;
do
{
while (continueLooping)
{
var feedUrl = GetPageUri(Feeds[_settings.Feed].BaseUrl, pageIndex);
await ProcessPageBlock.SendAsync(feedUrl).ConfigureAwait(false); // TODO: Need check with SongsPerPage
itemsInBlock++;
pageIndex++;
if (pageIndex > maxPages && useMaxPages)
{
continueLooping = false;
}
while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping)
{
if (itemsInBlock <= 0)
{
break;
}
await ProcessPageBlock.OutputAvailableAsync().ConfigureAwait(false);
while (ProcessPageBlock.TryReceive(out List <ScrapedSong> newSongs))
{
itemsInBlock--;
if (newSongs == null)
{
Logger.Debug("Received no new songs, last page reached.");
ProcessPageBlock.Complete();
itemsInBlock = 0;
continueLooping = false;
break;
}
Logger.Debug($"Receiving {newSongs.Count} potential songs from {newSongs.First().SourceUri}");
foreach (var song in newSongs)
{
if (retDict.ContainsKey(song.Hash))
{
Logger.Debug($"Song {song.Hash} already exists.");
}
else
{
if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0)
{
retDict.Add(song.Hash, song);
}
if (retDict.Count >= settings.MaxSongs && useMaxSongs)
{
continueLooping = false;
}
}
}
if (!useMaxPages || pageIndex <= maxPages)
{
if (retDict.Count < settings.MaxSongs)
{
continueLooping = true;
}
}
}
}
}
}while (continueLooping);
return(retDict);
}
/// <summary>High throughput parallel lazy-ish method</summary>
/// <remarks>
/// Inspired by https://stackoverflow.com/a/58564740/1128762
/// </remarks>
public static async IAsyncEnumerable <TResult> ParallelSelectAwait <TArg, TResult>(this IEnumerable <TArg> source,
Func <TArg, Task <TResult> > selector, int maxDop, [EnumeratorCancellation] CancellationToken token = default)
{
var processor = new TransformBlock <TArg, TResult>(selector, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDop,
BoundedCapacity = (maxDop * 5) / 4,
CancellationToken = token,
SingleProducerConstrained = true,
EnsureOrdered = false
});
bool pipelineTerminatedEarly = false;
foreach (var item in source)
{
while (!processor.Post(item))
{
var result = await ReceiveAsync();
if (pipelineTerminatedEarly)
{
break;
}
yield return(result);
}
if (pipelineTerminatedEarly)
{
break;
}
if (processor.TryReceive(out var resultIfAvailable))
{
yield return(resultIfAvailable);
}
}
processor.Complete();
while (await processor.OutputAvailableAsync(token))
{
var result = ReceiveKnownAvailable();
if (pipelineTerminatedEarly)
{
break;
}
yield return(result);
}
await processor.Completion;
if (pipelineTerminatedEarly)
{
throw new InvalidOperationException("Pipeline terminated early missing items, but no exception thrown");
}
async Task <TResult> ReceiveAsync()
{
await processor.OutputAvailableAsync();
return(ReceiveKnownAvailable());
}
TResult ReceiveKnownAvailable()
{
token.ThrowIfCancellationRequested();
if (!processor.TryReceive(out var item))
{
pipelineTerminatedEarly = true;
return(default);
// TODO: Abort early when bsaber.com is down (check if all items in block failed?)
// TODO: Make cancellationToken actually do something.
/// <summary>
/// Gets all songs from the feed defined by the provided settings.
/// </summary>
/// <param name="settings"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception>
/// <exception cref="ArgumentException">Thrown when trying to access a feed that requires a username and the username wasn't provided.</exception>
/// <exception cref="OperationCanceledException"></exception>
/// <returns></returns>
public async Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(FeedResult.CancelledResult);
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync.");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.FeedIndex != 2 && string.IsNullOrEmpty(_username?.Trim()))
{
//Logger?.Error($"Can't access feed without a valid username in the config file");
throw new ArgumentException("Cannot access this feed without a valid username.");
}
int pageIndex = settings.StartingPage;
int maxPages = _settings.MaxPages;
bool useMaxSongs = _settings.MaxSongs != 0;
bool useMaxPages = maxPages != 0;
if (useMaxPages && pageIndex > 1)
{
maxPages = maxPages + pageIndex - 1;
}
var ProcessPageBlock = new TransformBlock <Uri, PageReadResult>(async feedUri =>
{
Stopwatch sw = new Stopwatch();
sw.Start();
//Logger?.Debug($"Checking URL: {feedUrl}");
string pageText = "";
ContentType contentType = ContentType.Unknown;
string contentTypeStr = string.Empty;
IWebResponseMessage response = null;
try
{
response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
if ((response?.StatusCode ?? 500) == 500)
{
response?.Dispose();
response = null;
Logger?.Warning($"Internal server error on {feedUri}, retrying in 20 seconds");
await Task.Delay(20000).ConfigureAwait(false);
response = await WebUtils.WebClient.GetAsync(feedUri, cancellationToken).ConfigureAwait(false);
}
response.EnsureSuccessStatusCode();
contentTypeStr = response.Content.ContentType.ToLower();
if (ContentDictionary.ContainsKey(contentTypeStr))
{
contentType = ContentDictionary[contentTypeStr];
}
else
{
contentType = ContentType.Unknown;
}
pageText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
catch (WebClientException ex)
{
return(PageReadResult.FromWebClientException(ex, feedUri));
}
catch (OperationCanceledException)
{
return(new PageReadResult(feedUri, null, new FeedReaderException("Page read was cancelled.", new OperationCanceledException(), FeedReaderFailureCode.Cancelled), PageErrorType.Cancelled));
}
catch (Exception ex)
{
string message = $"Error downloading {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown));
}
finally
{
response?.Dispose();
response = null;
}
List <ScrapedSong> newSongs = null;
try
{
newSongs = GetSongsFromPageText(pageText, feedUri, contentType);
}
catch (JsonReaderException ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Error parsing page text for {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
catch (XmlException ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Error parsing page text for {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.ParsingError));
}
catch (Exception ex)
{
// TODO: Probably don't need a logger message here, caller can deal with it.
string message = $"Uncaught error parsing page text for {feedUri} in TransformBlock.";
Logger?.Debug(message);
Logger?.Debug($"{ex.Message}\n{ex.StackTrace}");
return(new PageReadResult(feedUri, null, new FeedReaderException(message, ex, FeedReaderFailureCode.PageFailed), PageErrorType.Unknown));
}
sw.Stop();
//Logger?.Debug($"Task for {feedUrl} completed in {sw.ElapsedMilliseconds}ms");
return(new PageReadResult(feedUri, newSongs));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxConcurrency,
BoundedCapacity = MaxConcurrency,
CancellationToken = cancellationToken
//#if NETSTANDARD
// , EnsureOrdered = true
//#endif
});
bool continueLooping = true;
int itemsInBlock = 0;
List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2);
do
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
while (continueLooping)
{
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
var feedUrl = GetPageUri(Feeds[_settings.Feed].BaseUrl, pageIndex);
await ProcessPageBlock.SendAsync(feedUrl, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage
itemsInBlock++;
pageIndex++;
if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
// TODO: Better http error handling, what if only a single page is broken and returns 0 songs?
while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping)
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
if (itemsInBlock <= 0)
{
break;
}
await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false);
while (ProcessPageBlock.TryReceive(out PageReadResult pageResult))
{
if (pageResult != null)
{
pageResults.Add(pageResult);
}
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
itemsInBlock--;
if (pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error.
{
Logger?.Debug("Received no new songs, last page reached.");
ProcessPageBlock.Complete();
itemsInBlock = 0;
continueLooping = false;
break;
}
if (pageResult.Count > 0)
{
Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}");
}
else
{
Logger?.Debug($"Did not find any songs in {Name}.{settings.FeedName}.");
}
// TODO: Process PageReadResults for better error feedback.
foreach (var song in pageResult.Songs)
{
if (!retDict.ContainsKey(song.Hash))
{
if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0)
{
retDict.Add(song.Hash, song);
}
if (retDict.Count >= settings.MaxSongs && useMaxSongs)
{
continueLooping = false;
}
}
}
if (!useMaxPages || pageIndex <= maxPages)
{
if (retDict.Count < settings.MaxSongs)
{
continueLooping = true;
}
}
}
}
}
}while (continueLooping);
return(new FeedResult(retDict, pageResults));
}
// TODO: Abort early when bsaber.com is down (check if all items in block failed?)
// TODO: Make cancellationToken actually do something.
/// <summary>
/// Gets all songs from the feed defined by the provided settings.
/// </summary>
/// <param name="settings"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="ArgumentNullException">Thrown when <paramref name="_settings"/> is null.</exception>
/// <exception cref="InvalidCastException">Thrown when the passed IFeedSettings isn't a BeastSaberFeedSettings.</exception>
/// <exception cref="OperationCanceledException"></exception>
/// <returns></returns>
public async override Task <FeedResult> GetSongsFromFeedAsync(IFeedSettings settings, IProgress <ReaderProgress> progress, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(FeedResult.CancelledResult);
}
if (settings == null)
{
throw new ArgumentNullException(nameof(settings), "settings cannot be null for BeastSaberReader.GetSongsFromFeedAsync.");
}
Dictionary <string, ScrapedSong> retDict = new Dictionary <string, ScrapedSong>();
if (!(settings is BeastSaberFeedSettings _settings))
{
throw new InvalidCastException(INVALIDFEEDSETTINGSMESSAGE);
}
if (_settings.Feed != BeastSaberFeedName.CuratorRecommended && string.IsNullOrEmpty(_settings.Username))
{
_settings.Username = Username;
}
BeastSaberFeed feed = new BeastSaberFeed(_settings)
{
StoreRawData = StoreRawData
};
try
{
feed.EnsureValidSettings();
}
catch (InvalidFeedSettingsException ex)
{
return(new FeedResult(null, null, ex, FeedResultError.Error));
}
int pageIndex = settings.StartingPage;
int maxPages = _settings.MaxPages;
int pagesChecked = 0;
bool useMaxSongs = _settings.MaxSongs != 0;
bool useMaxPages = maxPages != 0;
if (useMaxPages && pageIndex > 1)
{
maxPages = maxPages + pageIndex - 1;
}
var ProcessPageBlock = new TransformBlock <int, PageReadResult>(async pageNum =>
{
return(await feed.GetSongsFromPageAsync(pageNum, cancellationToken).ConfigureAwait(false));
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxConcurrency,
BoundedCapacity = MaxConcurrency,
CancellationToken = cancellationToken
//#if NETSTANDARD
// , EnsureOrdered = true
//#endif
});
bool continueLooping = true;
int itemsInBlock = 0;
List <PageReadResult> pageResults = new List <PageReadResult>(maxPages + 2);
do
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
while (continueLooping)
{
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
await ProcessPageBlock.SendAsync(pageIndex, cancellationToken).ConfigureAwait(false); // TODO: Need check with SongsPerPage
itemsInBlock++;
pageIndex++;
if ((pageIndex > maxPages && useMaxPages) || cancellationToken.IsCancellationRequested)
{
continueLooping = false;
}
// TODO: Better http error handling, what if only a single page is broken and returns 0 songs?
while (ProcessPageBlock.OutputCount > 0 || itemsInBlock == MaxConcurrency || !continueLooping)
{
if (cancellationToken.IsCancellationRequested)
{
continueLooping = false;
break;
}
if (itemsInBlock <= 0)
{
break;
}
await ProcessPageBlock.OutputAvailableAsync(cancellationToken).ConfigureAwait(false);
while (ProcessPageBlock.TryReceive(out PageReadResult pageResult))
{
int songsAdded = 0;
if (pageResult != null)
{
pageResults.Add(pageResult);
}
if (Utilities.IsPaused)
{
await Utilities.WaitUntil(() => !Utilities.IsPaused, 500, cancellationToken).ConfigureAwait(false);
}
itemsInBlock--;
if (pageResult.IsLastPage || pageResult == null || pageResult.Count == 0) // TODO: This will trigger if a single page has an error.
{
Logger?.Debug("Received no new songs, last page reached.");
ProcessPageBlock.Complete();
itemsInBlock = 0;
continueLooping = false;
break;
}
if (pageResult.Count > 0)
{
Logger?.Debug($"Receiving {pageResult.Count} potential songs from {pageResult.Uri}");
}
else
{
Logger?.Debug($"Did not find any songs on page '{pageResult.Uri}' of {Name}.{settings.FeedName}.");
}
// TODO: Process PageReadResults for better error feedback.
foreach (var song in pageResult.Songs)
{
if (!retDict.ContainsKey(song.Hash))
{
if (retDict.Count < settings.MaxSongs || settings.MaxSongs == 0)
{
retDict.Add(song.Hash, song);
songsAdded++;
}
if (retDict.Count >= settings.MaxSongs && useMaxSongs)
{
continueLooping = false;
}
}
}
int prog = Interlocked.Increment(ref pagesChecked);
progress?.Report(new ReaderProgress(prog, songsAdded));
if (!useMaxPages || pageIndex <= maxPages)
{
if (retDict.Count < settings.MaxSongs)
{
continueLooping = true;
}
}
}
}
}
}while (continueLooping);
if (pageResults.Any(r => r.PageError == PageErrorType.Cancelled))
{
return(FeedResult.GetCancelledResult(retDict, pageResults));
}
return(new FeedResult(retDict, pageResults));
}
public static async Task <IReadOnlyCollection <GraphTaskResult> > Run(this TaskGraph tasks, int parallel, ILogger log, CancellationToken cancel)
{
async Task <GraphTaskResult> RunTask(GraphTask task)
{
var sw = Stopwatch.StartNew();
GraphTaskResult Result(Exception ex = null) =>
new GraphTaskResult
{
Name = task.Name,
FinalStatus = task.Status,
Duration = sw.Elapsed,
Exception = ex
};
try {
if (cancel.IsCancellationRequested || tasks.DependenciesDeep(task).Any(d => d.Status.In(Cancelled, Error)))
{
task.Status = Cancelled;
return(Result());
}
task.Status = Running;
log = log.ForContext("Task", task.Name);
await task.Run(log, cancel);
if (cancel.IsCancellationRequested)
{
task.Status = Cancelled;
}
else
{
task.Status = Success;
}
return(Result());
}
catch (Exception ex) {
task.Status = Error;
log.Error(ex, "Task {Task} failed: {Message}", task.Name, ex.Message);
return(Result(ex));
}
}
var block = new TransformBlock <GraphTask, GraphTaskResult>(RunTask,
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = parallel
});
var newTaskSignal = new AsyncManualResetEvent(true);
async Task Producer()
{
while (!tasks.AllComplete)
{
if (cancel.IsCancellationRequested)
{
foreach (var t in tasks.All.Where(t => t.Status.IsIncomplete()))
{
t.Status = Cancelled;
}
}
var tasksToAdd = tasks.AvailableToRun().ToList();
if (tasksToAdd.IsEmpty())
{
// if no tasks are ready to start. Wait to either be signaled, or log which tasks are still running
var logTimeTask = Task.Delay(1.Minutes(), cancel);
await Task.WhenAny(logTimeTask, newTaskSignal.WaitAsync());
if (newTaskSignal.IsSet)
{
newTaskSignal.Reset();
}
if (logTimeTask.IsCompleted)
{
log.Debug("Waiting for {TaskList} to complete", tasks.Running.Select(t => t.Name));
}
}
foreach (var task in tasksToAdd)
{
task.Status = Queued;
await block.SendAsync(task);
}
}
block.Complete();
}
var producer = Producer();
var taskResults = new List <GraphTaskResult>();
while (await block.OutputAvailableAsync())
{
var item = await block.ReceiveAsync();
taskResults.Add(item);
newTaskSignal.Set();
}
await Task.WhenAll(producer, block.Completion);
return(taskResults);
}
