/// <summary>
/// Splits a stream into many streams that can be independently caught up.
/// </summary>
/// <typeparam name="TUpstream">The type of the upstream stream.</typeparam>
/// <typeparam name="TDownstream">The type of the downstream stream.</typeparam>
/// <typeparam name="TUpstreamCursor">The type of the upstream cursor.</typeparam>
/// <param name="upstream">The upstream.</param>
/// <param name="queryDownstream">The query downstream.</param>
/// <returns></returns>
public static IStream <TDownstream, TUpstreamCursor> IntoMany <TUpstream, TDownstream, TUpstreamCursor>(
this IStream <TUpstream, TUpstreamCursor> upstream,
QueryDownstream <TUpstream, TDownstream, TUpstreamCursor> queryDownstream)
{
return(Create(
id: string.Format("{0}->IntoMany(d:{1})", upstream.Id, typeof(TDownstream).ReadableName()),
query: async upstreamQuery =>
{
var upstreamBatch = await upstream.Fetch(
upstream.CreateQuery(upstreamQuery.Cursor,
upstreamQuery.BatchSize));
var streams = upstreamBatch.Select(
async x =>
{
TUpstreamCursor startingCursor = upstreamBatch.StartsAtCursorPosition;
return await queryDownstream(x,
startingCursor,
upstreamQuery.Cursor.Position);
});
return await streams.AwaitAll();
},
advanceCursor: (query, batch) =>
{
// we're passing the cursor through to the upstream query, so we don't want downstream queries to overwrite it
},
newCursor: upstream.NewCursor));
}
/// <summary>
/// Maps data from a stream into a new form.
/// </summary>
public static IStream <TTo, TCursor> Map <TFrom, TTo, TCursor>(
this IStream <TFrom, TCursor> source,
Func <IEnumerable <TFrom>, IEnumerable <TTo> > map,
string id = null)
{
return(Create <TTo, TCursor>(
id: id ?? string.Format("{0}->Map(d:{1})", source.Id, typeof(TTo).ReadableName()),
query: async q =>
{
var query = source.CreateQuery(q.Cursor, q.BatchSize);
var sourceBatch = await source.Fetch(query);
var mappedItems = map(sourceBatch);
var mappedCursor = Cursor.New <TCursor>(sourceBatch.StartsAtCursorPosition);
var mappedBatch = StreamBatch.Create(mappedItems, mappedCursor);
return mappedBatch;
},
advanceCursor: (query, batch) =>
{
// don't advance the cursor in the map operation, since sourceStream.Fetch will already have done it
},
newCursor: source.NewCursor));
}
/// <summary>
/// Aggregates a single batch of data from a stream using the specified aggregator and projection.
/// </summary>
/// <typeparam name="TProjection">The type of the projection.</typeparam>
/// <typeparam name="TData">The type of the data.</typeparam>
/// <typeparam name="TCursor">The type of the cursor.</typeparam>
/// <param name="stream">The stream.</param>
/// <param name="aggregator">The aggregator.</param>
/// <param name="projection">The projection.</param>
/// <returns>The updated state of the projection.</returns>
/// <remarks>This method can be used to create on-demand projections. It does not do any projection persistence.</remarks>
public static async Task <TProjection> Aggregate <TProjection, TData, TCursor>(
this IStream <TData, TCursor> stream,
IStreamAggregator <TProjection, TData> aggregator,
TProjection projection = null)
where TProjection : class
{
var cursor = (projection as ICursor <TCursor>) ??
stream.NewCursor();
var query = stream.CreateQuery(cursor);
var data = await query.NextBatch();
if (data.Any())
{
projection = await aggregator.Aggregate(projection, data);
}
return(projection);
}
protected async Task <ICursor <TCursor> > RunSingleBatch <TCursor>(
IStream <TData, TCursor> stream,
ICursor <TCursor> initialCursor = null)
{
TaskCompletionSource <AggregationBatch <TCursor> > tcs = null;
tcs = new TaskCompletionSource <AggregationBatch <TCursor> >();
var exchange = Interlocked.CompareExchange <object>(ref batchTaskCompletionSource, tcs, null);
if (exchange != null)
{
Debug.WriteLine("[Catchup] RunSingleBatch returning early");
var batch = await((TaskCompletionSource <AggregationBatch <TCursor> >)batchTaskCompletionSource).Task;
return(batch.Cursor);
}
ICursor <TCursor> upstreamCursor = null;
var projections = new ConcurrentBag <object>();
Action runQuery = async() =>
{
var cursor = initialCursor ?? projections.OfType <ICursor <TCursor> >().MinOrDefault();
upstreamCursor = cursor;
var query = stream.CreateQuery(cursor, batchSize);
try
{
var batch = await query.NextBatch();
tcs.SetResult(new AggregationBatch <TCursor>
{
Cursor = query.Cursor,
Batch = batch
});
}
catch (Exception exception)
{
tcs.SetException(exception);
}
};
Func <object, Task <AggregationBatch <TCursor> > > awaitData = c =>
{
projections.Add(c);
if (projections.Count >= aggregatorSubscriptions.Count)
{
runQuery();
}
return(tcs.Task);
};
var aggregationTasks = aggregatorSubscriptions
.Values
.Select(v => Aggregate(stream, (dynamic)v, awaitData) as Task);
await Task.WhenAll(aggregationTasks);
this.batchTaskCompletionSource = null;
return(upstreamCursor);
}