public void SetUp()
{
ints = Enumerable.Range(1, 1000).ToArray();
disposables = new CompositeDisposable();
partitionedStream = Stream
.Partitioned <int, int, int>(
query: async(q, p) =>
{
return(ints
.Where(i => i.IsWithinPartition(p))
.Skip(q.Cursor.Position)
.Take(q.BatchSize.Value));
},
advanceCursor: (query, batch) =>
{
// putting the cursor and the partition on the same field is a little weird because a batch of zero doesn't necessarily signify the end of the batch
if (batch.Any())
{
query.Cursor.AdvanceTo(batch.Last());
}
});
Formatter.ListExpansionLimit = 100;
Formatter <Projection <HashSet <int>, int> > .RegisterForAllMembers();
}
public static IPartitionedStream <TData, TCursor, TPartition> Trace <TData, TCursor, TPartition>(
this IPartitionedStream <TData, TCursor, TPartition> stream)
{
return(new AnonymousPartitionedStream <TData, TCursor, TPartition>(
stream.Id,
async p => (await stream.GetStream(p)).Trace()));
}
public void SetUp()
{
ints = Enumerable.Range(1, 1000).ToArray();
disposables = new CompositeDisposable();
partitionedStream = Stream
.Partitioned<int, int, int>(
query: async (q, p) =>
{
return ints
.Where(i => i.IsWithinPartition(p))
.Skip(q.Cursor.Position)
.Take(q.BatchSize.Value);
},
advanceCursor: (query, batch) =>
{
// putting the cursor and the partition on the same field is a little weird because a batch of zero doesn't necessarily signify the end of the batch
if (batch.Any())
{
query.Cursor.AdvanceTo(batch.Last());
}
});
Formatter.ListExpansionLimit = 100;
Formatter<Projection<HashSet<int>, int>>.RegisterForAllMembers();
}
/// <summary>
/// Maps data from a stream into a new form.
/// </summary>
public static IPartitionedStream <TTo, TCursor, TPartition> Map <TFrom, TTo, TCursor, TPartition>(
this IPartitionedStream <TFrom, TCursor, TPartition> source,
Func <IEnumerable <TFrom>, IEnumerable <TTo> > map,
string id = null)
{
return(new AnonymousPartitionedStream <TTo, TCursor, TPartition>(
id: id,
getStream: async partition =>
{
var stream = await source.GetStream(partition);
return stream.Map(map);
}));
}
/// <summary>
/// Distributes a stream catchup the among one or more partitions using a specified distributor.
/// </summary>
/// <remarks>If no distributor is provided, then distribution is done in-process.</remarks>
public static IStreamCatchup <TData, TCursor> DistributeAmong <TData, TCursor, TPartition>(
this IPartitionedStream <TData, TCursor, TPartition> streams,
IEnumerable <IStreamQueryPartition <TPartition> > partitions,
int?batchSize = null,
FetchAndSaveProjection <ICursor <TCursor> > cursorPerPartition = null,
IDistributor <IStreamQueryPartition <TPartition> > distributor = null)
{
return(new DistributedStreamCatchup <TData, TCursor, TPartition>(
streams,
partitions,
batchSize,
cursorPerPartition,
distributor));
}
public DistributedStreamCatchup(
IPartitionedStream <TData, TCursor, TPartition> partitionedStream,
IEnumerable <IStreamQueryPartition <TPartition> > partitions,
int?batchSize = null,
FetchAndSaveProjection <ICursor <TCursor> > manageCursor = null,
IDistributor <IStreamQueryPartition <TPartition> > distributor = null) : base(batchSize)
{
if (partitionedStream == null)
{
throw new ArgumentNullException("partitionedStream");
}
if (partitions == null)
{
throw new ArgumentNullException("partitions");
}
this.partitionedStream = partitionedStream;
this.distributor = distributor ?? partitions.DistributeQueriesInProcess();
manageCursor = manageCursor ?? ((id, aggregate) => aggregate(null));
this.distributor
#if DEBUG
.Trace() // TODO: (DistributedStreamCatchup) figure out a way to let people Trace this distributor
#endif
.OnReceive(async lease =>
{
await manageCursor(lease.ResourceName, async cursor =>
{
var catchup = new SingleStreamCatchup <TData, TCursor>(
await partitionedStream.GetStream(lease.Resource),
initialCursor: cursor,
batchSize: batchSize,
subscriptions: new ConcurrentDictionary <Type, IAggregatorSubscription>(aggregatorSubscriptions));
return(await catchup.RunSingleBatch());
});
});
}
public void SetUp()
{
words = Values.AtoZ().SelectMany(c => Enumerable.Range(1, 100).Select(i => c + i)).ToList();
partitions = Values.AtoZ().Select(c => Partition.Where <string>(s => s.StartsWith(c), named: c));
partitionedStream = Stream
.Partitioned <string, int, string>(
query: async(q, partition) =>
{
var wordsInPartition = words
.Skip(q.Cursor.Position)
.Where(partition.Contains);
var b = wordsInPartition
.Take(q.BatchSize.Value);
return(b);
},
advanceCursor: (query, batch) => { query.Cursor.AdvanceTo(words.IndexOf(batch.Last()) + 1); });
Formatter.ListExpansionLimit = 100;
Formatter <Projection <HashSet <int>, int> > .RegisterForAllMembers();
}
public void SetUp()
{
words = Values.AtoZ().SelectMany(c => Enumerable.Range(1, 100).Select(i => c + i)).ToList();
partitions = Values.AtoZ().Select(c => Partition.Where<string>(s => s.StartsWith(c), named: c));
partitionedStream = Stream
.Partitioned<string, int, string>(
query: async (q, partition) =>
{
var wordsInPartition = words
.Skip(q.Cursor.Position)
.Where(partition.Contains);
var b = wordsInPartition
.Take(q.BatchSize.Value);
return b;
},
advanceCursor: (query, batch) => { query.Cursor.AdvanceTo(words.IndexOf(batch.Last()) + 1); });
Formatter.ListExpansionLimit = 100;
Formatter<Projection<HashSet<int>, int>>.RegisterForAllMembers();
}
