public CallbackReader(IAsyncEnumerable<ISegmentReader> segmentReaders, Action<WorkBuffer> enqueue, IBlockingPool<WorkBuffer> bufferPool)
{
if (null == segmentReaders)
throw new ArgumentNullException(nameof(segmentReaders));
if (null == enqueue)
throw new ArgumentNullException(nameof(enqueue));
if (null == bufferPool)
throw new ArgumentNullException(nameof(bufferPool));
_segmentReaders = segmentReaders;
_enqueue = enqueue;
_bufferPool = bufferPool;
}
public static Task <double> Max <T>(this IAsyncEnumerable <T> enumerable, Func <T, Task <double> > selector)
{
return(MaxCore(enumerable.Select(selector), (current, next) => next > current));
}
public static async Task <decimal> Max(this IAsyncEnumerable <decimal?> enumerable)
{
return((await MaxCore(enumerable, (current, next) => next.HasValue && next.Value > current).ConfigureAwait(false)).Value);
}
public CancelableTypedAsyncEnumerable(IAsyncEnumerable <TResult> asyncEnumerable, CancellationTokenSource cts)
{
_asyncEnumerable = asyncEnumerable;
_cts = cts;
}
public static IAsyncEnumerable <object> MakeCancelableAsyncEnumerable <T>(IAsyncEnumerable <T> asyncEnumerable, CancellationToken cancellationToken = default)
{
return(new CancelableAsyncEnumerable <T>(asyncEnumerable, cancellationToken));
}
internal static (IReadOnlyList <T> Elements, bool Finished) TearOffPrefetchedElements <T>(this IAsyncEnumerable <T> enumerable)
{
internal FdbQueryAsyncEnumerableExpression(IAsyncEnumerable <T> source)
{
Contract.Requires(source != null);
this.Source = source;
}
public async Task ReceivingMixOfSyncAndAsyncMessageFromTorControlAsync()
{
using CancellationTokenSource timeoutCts = new(TimeSpan.FromSeconds(120));
// Test parameters.
const string AsyncEventContent = "CIRC 1000 EXTENDED moria1,moria2";
Pipe toServer = new();
Pipe toClient = new();
// Set up Tor control client.
await using TorControlClient client = new(pipeReader : toClient.Reader, pipeWriter : toServer.Writer);
// Subscribe to Tor events.
IAsyncEnumerable <TorControlReply> events = client.ReadEventsAsync(timeoutCts.Token);
IAsyncEnumerator <TorControlReply> eventsEnumerator = events.GetAsyncEnumerator();
ValueTask <bool> firstReplyTask = eventsEnumerator.MoveNextAsync();
Task serverTask = Task.Run(async() =>
{
Logger.LogTrace($"Server: Send msg #1 (async) to client: '650 {AsyncEventContent}'.");
await toClient.Writer.WriteAsciiAndFlushAsync($"650 {AsyncEventContent}\r\n", timeoutCts.Token).ConfigureAwait(false);
Logger.LogTrace($"Server: Send msg #2 (async) to client: '650 {AsyncEventContent}'.");
await toClient.Writer.WriteAsciiAndFlushAsync($"650 {AsyncEventContent}\r\n", timeoutCts.Token).ConfigureAwait(false);
Logger.LogTrace("Server: Wait for TAKEOWNERSHIP command.");
string command = await toServer.Reader.ReadLineAsync(timeoutCts.Token).ConfigureAwait(false);
Assert.Equal("TAKEOWNERSHIP", command);
Logger.LogTrace("Server: Send msg #3 (sync) to client in response to TAKEOWNERSHIP command.");
await toClient.Writer.WriteAsciiAndFlushAsync($"250 OK\r\n", timeoutCts.Token).ConfigureAwait(false);
Logger.LogTrace($"Server: Send msg #4 (async) to client: '650 {AsyncEventContent}'.");
await toClient.Writer.WriteAsciiAndFlushAsync($"650 {AsyncEventContent}\r\n", timeoutCts.Token).ConfigureAwait(false);
});
Logger.LogTrace("Client: Receive msg #1 (async).");
{
await firstReplyTask.AsTask().WithAwaitCancellationAsync(timeoutCts.Token);
TorControlReply receivedEvent1 = eventsEnumerator.Current;
Assert.Equal(StatusCode.AsynchronousEventNotify, receivedEvent1.StatusCode);
string line = Assert.Single(receivedEvent1.ResponseLines);
Assert.Equal(AsyncEventContent, line);
}
// Msg #2 is sent before expected reply (msg #3).
TorControlReply takeOwnershipReply = await client.TakeOwnershipAsync(timeoutCts.Token);
Assert.True(takeOwnershipReply.Success);
Logger.LogTrace("Client: Receive msg #2 (async).");
{
Assert.True(await eventsEnumerator.MoveNextAsync());
TorControlReply receivedEvent2 = eventsEnumerator.Current;
Assert.Equal(StatusCode.AsynchronousEventNotify, receivedEvent2.StatusCode);
string line = Assert.Single(receivedEvent2.ResponseLines);
Assert.Equal(AsyncEventContent, line);
}
Logger.LogTrace("Client: Receive msg #4 (async) - i.e. third async event.");
{
Assert.True(await eventsEnumerator.MoveNextAsync());
TorControlReply receivedEvent3 = eventsEnumerator.Current;
Assert.Equal(StatusCode.AsynchronousEventNotify, receivedEvent3.StatusCode);
string line = Assert.Single(receivedEvent3.ResponseLines);
Assert.Equal(AsyncEventContent, line);
}
// Client decided to stop reading Tor async events.
timeoutCts.Cancel();
// No more async events.
_ = Assert.ThrowsAsync <OperationCanceledException>(async() => await eventsEnumerator.MoveNextAsync());
}
/// <summary>
/// Explicitly load the data by id dictionary.
/// </summary>
/// <param name="dataById">Dictionary to load</param>
/// <param name="transaction">Transaction to use</param>
/// <param name="cancellation">Cancellation token to indicate the cancellation of loading</param>
/// <returns>A <see cref="Task"/> that tracks execution of this method</returns>
private async Task LoadDictionary(
IReliableDictionary <long, WinFabPersistence.PersistedData> dataById,
ITransaction transaction,
CancellationToken cancellation)
{
ServiceFabricPersistenceEventSource.Log.LoadDictionaryStarted();
Stopwatch timer = Stopwatch.StartNew();
var dataList = new List <PersistedData>();
try
{
ServiceFabricPersistenceEventSource.Log.LoadDictionaryStartEnumeration(transaction.TransactionId);
IAsyncEnumerable <KeyValuePair <long, WinFabPersistence.PersistedData> > enumerable = await dataById.CreateEnumerableAsync(transaction);
using (var enumerator = enumerable.GetAsyncEnumerator())
{
while (await enumerator.MoveNextAsync(cancellation))
{
dataList.Add(enumerator.Current.Value.Data);
}
}
this.Load(dataList, this.configuration.IgnoreErrorsDuringLoad);
ServiceFabricPersistenceEventSource.Log.LoadDictionaryCompleted(dataList.Count, timer.ElapsedMilliseconds);
}
catch (Exception ex)
{
ServiceFabricPersistenceEventSource.Log.LoadDictionaryFailed(dataList.Count, timer.ElapsedMilliseconds, ex.ToString());
this.ReportFatalError("Failed to load dictionary", ex);
throw;
}
}
public static IAsyncEnumerable <TResult> Create <TOuter, TInner, TKey, TResult>(IAsyncEnumerable <TOuter> outer, IAsyncEnumerable <TInner> inner, Func <TOuter, TKey> outerKeySelector, Func <TInner, TKey> innerKeySelector, Func <TOuter, IEnumerable <TInner>, TResult> resultSelector) => new AsyncGroupJoinEnumerable <TOuter, TInner, TKey>(outer, inner.AsEnumerable(), outerKeySelector, innerKeySelector) .Select(result => resultSelector.Invoke(result.outer, result.inner));
public static IAsyncEnumerable <TResult> Create <TOuter, TInner, TKey, TResult>(IAsyncEnumerable <TOuter> outer, Task <IEnumerable <TInner> > inner, Func <TOuter, TKey> outerKeySelector, Func <TInner, TKey> innerKeySelector, Func <TOuter, TInner, TResult> resultSelector) => new AsyncGroupJoinEnumerable <TOuter, TInner, TKey>(outer, inner, outerKeySelector, innerKeySelector) .SelectMany(result => result.inner.Select(value => resultSelector.Invoke(result.outer, value)));
public static IAsyncEnumerable <TSource> DistinctUntilChanged <TSource, TKey>(this IAsyncEnumerable <TSource> source, Func <TSource, TKey> keySelector)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
if (keySelector == null)
{
throw new ArgumentNullException(nameof(keySelector));
}
return(source.DistinctUntilChanged_(keySelector, EqualityComparer <TKey> .Default));
}
private static IAsyncEnumerable <TSource> DistinctUntilChanged_ <TSource, TKey>(this IAsyncEnumerable <TSource> source, Func <TSource, TKey> keySelector, IEqualityComparer <TKey> comparer)
{
return(new DistinctUntilChangedAsyncIterator <TSource, TKey>(source, keySelector, comparer));
}
protected AniDroidRecyclerAdapter(BaseAniDroidActivity context,
AniDroidRecyclerAdapter <T, TModel> adapter) : base(context, adapter.Items, adapter.CardType)
{
_asyncEnumerable = adapter._asyncEnumerable;
_asyncEnumerator = adapter._asyncEnumerator;
}
private Task PopulateStream(int minRevision, int maxRevision, IAsyncEnumerable<ICommit> commits)
{
commits = commits ?? AsyncEnumerable.Empty<ICommit>();
return commits.ForEach(context =>
{
var commit = context.Item;
Logger.Verbose(Resources.AddingCommitsToStream, commit.CommitId, commit.Events.Count, StreamId);
_identifiers.Add(commit.CommitId);
CommitSequence = commit.CommitSequence;
int currentRevision = commit.StreamRevision - commit.Events.Count + 1;
if (currentRevision > maxRevision)
{
context.Break();
return Task.FromResult(false);
}
CopyToCommittedHeaders(commit);
CopyToEvents(minRevision, maxRevision, currentRevision, commit);
return Task.FromResult(false);
});
}
protected AppendPrependAsyncIterator(IAsyncEnumerable <TSource> source)
{
_source = source;
}
public ObserverAsyncIterator(IAsyncEnumerable <TSource> source, AsyncObserverExpression <TSource> observer)
: base(source)
{
Contract.Requires(observer != null);
m_observer = observer;
}
protected override async Task <BulkCopyRowsCopied> MultipleRowsCopyAsync <T>(
ITable <T> table, BulkCopyOptions options, IAsyncEnumerable <T> source, CancellationToken cancellationToken)
{
await using (new InvariantCultureRegion(null))
return(await base.MultipleRowsCopyAsync(table, options, source, cancellationToken).ConfigureAwait(Configuration.ContinueOnCapturedContext));
}
public static async Task Paginate <T>(this Context ctx, IAsyncEnumerable <T> items, int totalCount, int itemsPerPage, string title, Func <DiscordEmbedBuilder, IEnumerable <T>, Task> renderer)
{
// TODO: make this generic enough we can use it in Choose<T> below
var buffer = new List <T>();
await using var enumerator = items.GetAsyncEnumerator();
var pageCount = (int)Math.Ceiling(totalCount / (double)itemsPerPage);
async Task <DiscordEmbed> MakeEmbedForPage(int page)
{
var bufferedItemsNeeded = (page + 1) * itemsPerPage;
while (buffer.Count < bufferedItemsNeeded && await enumerator.MoveNextAsync())
{
buffer.Add(enumerator.Current);
}
var eb = new DiscordEmbedBuilder();
eb.Title = pageCount > 1 ? $"[{page+1}/{pageCount}] {title}" : title;
await renderer(eb, buffer.Skip(page *itemsPerPage).Take(itemsPerPage));
return(eb.Build());
}
try
{
var msg = await ctx.Reply(embed : await MakeEmbedForPage(0));
if (pageCount <= 1)
{
return; // If we only have one (or no) page, don't bother with the reaction/pagination logic, lol
}
string[] botEmojis = { "\u23EA", "\u2B05", "\u27A1", "\u23E9", Emojis.Error };
var _ = msg.CreateReactionsBulk(botEmojis); // Again, "fork"
try {
var currentPage = 0;
while (true)
{
var reaction = await ctx.AwaitReaction(msg, ctx.Author, timeout : TimeSpan.FromMinutes(5));
// Increment/decrement page counter based on which reaction was clicked
if (reaction.Emoji.Name == "\u23EA")
{
currentPage = 0; // <<
}
if (reaction.Emoji.Name == "\u2B05")
{
currentPage = (currentPage - 1) % pageCount; // <
}
if (reaction.Emoji.Name == "\u27A1")
{
currentPage = (currentPage + 1) % pageCount; // >
}
if (reaction.Emoji.Name == "\u23E9")
{
currentPage = pageCount - 1; // >>
}
if (reaction.Emoji.Name == Emojis.Error)
{
break; // X
}
// C#'s % operator is dumb and wrong, so we fix negative numbers
if (currentPage < 0)
{
currentPage += pageCount;
}
// If we can, remove the user's reaction (so they can press again quickly)
if (ctx.BotHasAllPermissions(Permissions.ManageMessages))
{
await msg.DeleteReactionAsync(reaction.Emoji, reaction.User);
}
// Edit the embed with the new page
var embed = await MakeEmbedForPage(currentPage);
await msg.ModifyAsync(embed : embed);
}
} catch (TimeoutException) {
// "escape hatch", clean up as if we hit X
}
if (ctx.BotHasAllPermissions(Permissions.ManageMessages))
{
await msg.DeleteAllReactionsAsync();
}
}
// If we get a "NotFound" error, the message has been deleted and thus not our problem
catch (NotFoundException) { }
}
/// <summary>
/// Extracts the <see cref="JsonRpcEnumerableSettings"/> from an <see cref="IAsyncEnumerable{T}"/>
/// that may have been previously returned from <see cref="WithJsonRpcSettings{T}(IAsyncEnumerable{T}, JsonRpcEnumerableSettings?)"/>.
/// </summary>
/// <typeparam name="T">The type of element enumerated by the sequence.</typeparam>
/// <param name="enumerable">The enumerable, which may have come from <see cref="WithJsonRpcSettings{T}(IAsyncEnumerable{T}, JsonRpcEnumerableSettings?)"/>.</param>
/// <returns>The settings to use.</returns>
/// <remarks>
/// If the <paramref name="enumerable"/> did not come from <see cref="WithJsonRpcSettings{T}(IAsyncEnumerable{T}, JsonRpcEnumerableSettings?)"/>,
/// the default settings will be returned.
/// </remarks>
internal static JsonRpcEnumerableSettings GetJsonRpcSettings <T>(this IAsyncEnumerable <T> enumerable)
{
Requires.NotNull(enumerable, nameof(enumerable));
return((enumerable as RpcEnumerable <T>)?.Settings ?? JsonRpcEnumerableSettings.DefaultSettings);
}
private static async Task <TResult> Aggregate_ <TSource, TAccumulate, TResult>(IAsyncEnumerable <TSource> source, TAccumulate seed,
Func <TAccumulate, TSource, TAccumulate> accumulator, Func <TAccumulate, TResult> resultSelector, CancellationToken cancellationToken)
{
var acc = seed;
using (var e = source.GetEnumerator())
{
while (await e.MoveNext(cancellationToken)
.ConfigureAwait(false))
{
acc = accumulator(acc, e.Current);
}
}
return(resultSelector(acc));
}
public StreamReadResults(IAsyncEnumerable <StoredEvent> events, bool streamExists, StreamPosition streamPosition)
{
this.events = events ?? throw new ArgumentNullException(nameof(events));
this.streamExists = streamExists;
this.streamPosition = streamPosition;
}
public RecommendationEdgeRecyclerAdapter(BaseAniDroidActivity context, IAsyncEnumerable <OneOf <IPagedData <ConnectionEdge <Recommendation> >, IAniListError> > enumerable, RecyclerCardType cardType, Func <ConnectionEdge <Recommendation>, RecommendationEdgeViewModel> createViewModelFunc) : base(context, enumerable, cardType, createViewModelFunc)
{
SetDefaultClickActions();
ValidateItemFunc = rec => rec.Node?.MediaRecommendation != null;
}
public static IAsyncEnumerable <T> MakeCancelableTypedAsyncEnumerable <T>(IAsyncEnumerable <T> asyncEnumerable, CancellationTokenSource cts)
{
return(new CancelableTypedAsyncEnumerable <T>(asyncEnumerable, cts));
}
private async Task StreamResultsAsync(string invocationId, HubConnectionContext connection, IAsyncEnumerable <object> enumerable, IServiceScope scope,
IHubActivator <THub> hubActivator, THub hub, CancellationTokenSource streamCts, HubMethodInvocationMessage hubMethodInvocationMessage)
{
string error = null;
try
{
await foreach (var streamItem in enumerable)
{
// Send the stream item
await connection.WriteAsync(new StreamItemMessage(invocationId, streamItem));
}
}
catch (ChannelClosedException ex)
{
// If the channel closes from an exception in the streaming method, grab the innerException for the error from the streaming method
error = ErrorMessageHelper.BuildErrorMessage("An error occurred on the server while streaming results.", ex.InnerException ?? ex, _enableDetailedErrors);
}
catch (Exception ex)
{
// If the streaming method was canceled we don't want to send a HubException message - this is not an error case
if (!(ex is OperationCanceledException && connection.ActiveRequestCancellationSources.TryGetValue(invocationId, out var cts) &&
cts.IsCancellationRequested))
{
error = ErrorMessageHelper.BuildErrorMessage("An error occurred on the server while streaming results.", ex, _enableDetailedErrors);
}
}
finally
{
await CleanupInvocation(connection, hubMethodInvocationMessage, hubActivator, hub, scope);
// Dispose the linked CTS for the stream.
streamCts.Dispose();
await connection.WriteAsync(CompletionMessage.WithError(invocationId, error));
if (connection.ActiveRequestCancellationSources.TryRemove(invocationId, out var cts))
{
cts.Dispose();
}
}
}
public CancelableAsyncEnumerable(IAsyncEnumerable <T> asyncEnumerable, CancellationToken cancellationToken)
{
_asyncEnumerable = asyncEnumerable;
_cancellationToken = cancellationToken;
}
public ReverseAsyncIterator(IAsyncEnumerable <TSource> source)
{
_source = source;
}
public static Task <int> Max(this IAsyncEnumerable <int> enumerable)
{
return(MaxCore(enumerable, (current, next) => next > current));
}
/// <summary> /// Produces the set difference of two async-enumerable sequences by using the default equality comparer to compare values. /// </summary> /// <typeparam name="TSource">The type of the elements of the input sequences.</typeparam> /// <param name="first">An async-enumerable sequence whose elements that are not also in second will be returned.</param> /// <param name="second">An async-enumerable sequence whose elements that also occur in the first sequence will cause those elements to be removed from the returned sequence.</param> /// <returns>A sequence that contains the set difference of the elements of two sequences.</returns> /// <exception cref="ArgumentNullException"><paramref name="first"/> or <paramref name="second"/> is null</exception> public static IAsyncEnumerable <TSource> Except <TSource>(this IAsyncEnumerable <TSource> first, IAsyncEnumerable <TSource> second) => Except(first, second, comparer: null);
public static async Task <double> Max <T>(this IAsyncEnumerable <T> enumerable, Func <T, Task <double?> > selector)
{
return((await MaxCore(enumerable.Select(selector), (current, next) => next.HasValue && next.Value > current).ConfigureAwait(false)).Value);
}
/// <summary>
/// Produces the set difference of two async-enumerable sequences by using the specified equality comparer to compare values.
/// </summary>
/// <typeparam name="TSource">The type of the elements of the input sequences.</typeparam>
/// <param name="first">An async-enumerable sequence whose elements that are not also in second will be returned.</param>
/// <param name="second">An async-enumerable sequence whose elements that also occur in the first sequence will cause those elements to be removed from the returned sequence.</param>
/// <param name="comparer">An equality comparer to compare values.</param>
/// <returns>A sequence that contains the set difference of the elements of two sequences.</returns>
/// <exception cref="ArgumentNullException"><paramref name="first"/> or <paramref name="second"/> is null.</exception>
public static IAsyncEnumerable <TSource> Except <TSource>(this IAsyncEnumerable <TSource> first, IAsyncEnumerable <TSource> second, IEqualityComparer <TSource>?comparer)
{
if (first == null)
{
throw Error.ArgumentNull(nameof(first));
}
if (second == null)
{
throw Error.ArgumentNull(nameof(second));
}
return(Create(Core));
async IAsyncEnumerator <TSource> Core(CancellationToken cancellationToken)
{
var set = new Set <TSource>(comparer);
await foreach (var element in second.WithCancellation(cancellationToken).ConfigureAwait(false))
{
set.Add(element);
}
await foreach (var element in first.WithCancellation(cancellationToken).ConfigureAwait(false))
{
if (set.Add(element))
{
yield return(element);
}
}
}
}
public async Task ReceiveTorAsyncEventsUsingForeachAsync()
{
using CancellationTokenSource timeoutCts = new(TimeSpan.FromMinutes(3));
// Test parameters.
const int ExpectedEventsNo = 3;
const string AsyncEventContent = "CIRC 1000 EXTENDED moria1,moria2";
Pipe toServer = new();
Pipe toClient = new();
// Set up Tor control client.
await using TorControlClient client = new(pipeReader : toClient.Reader, pipeWriter : toServer.Writer);
// Subscribe to Tor events.
IAsyncEnumerable <TorControlReply> events = client.ReadEventsAsync(timeoutCts.Token);
// Send a Tor event to all subscribed clients (only one here).
// This must happen after a client is subscribed.
Task serverTask = Task.Run(async() =>
{
// We do not want to send the data until the client is really subscribed.
while (!timeoutCts.IsCancellationRequested)
{
if (client.SubscriberCount == 1)
{
break;
}
await Task.Delay(200).ConfigureAwait(false);
}
for (int i = 0; i < ExpectedEventsNo; i++)
{
Logger.LogTrace($"Server: Send async Tor event (#{i}): '650 {AsyncEventContent}'.");
await toClient.Writer.WriteAsciiAndFlushAsync($"650 {AsyncEventContent}\r\n", timeoutCts.Token).ConfigureAwait(false);
}
});
// Iterate received events.
int counter = 0;
// Client should get all the events.
await foreach (TorControlReply receivedEvent in events)
{
counter++;
Logger.LogTrace($"Client: Received event (#{counter}): '{receivedEvent}'.");
Assert.Equal(StatusCode.AsynchronousEventNotify, receivedEvent.StatusCode);
string line = Assert.Single(receivedEvent.ResponseLines);
Assert.Equal(AsyncEventContent, line);
if (counter == ExpectedEventsNo)
{
Assert.Equal(1, client.SubscriberCount);
break;
}
}
// Verifies that "break" in "await foreach" actually removes the internal Channel<T> (subscription).
Assert.Equal(0, client.SubscriberCount);
Logger.LogTrace("Client: Done.");
}
private IAsyncEnumerable<ICommit> ExecuteHooks(IAsyncEnumerable<ICommit> commits)
{
return AsyncEnumerable.Create<ICommit>(async producer =>
{
//List<ICommit> results = new List<ICommit>();
await commits.ForEach(async context =>
{
var commit = context.Item;
var filtered = commit;
foreach (var hook in _pipelineHooks)
{
filtered = await hook.Select(filtered);
if (filtered == null)
{
Logger.Info(Resources.PipelineHookSkippedCommit, hook.GetType(), commit.CommitId);
break;
}
}
if (filtered == null)
{
Logger.Info(Resources.PipelineHookFilteredCommit);
}
else
{
await producer.Yield(filtered);
}
});
//return results;
});
}