public void SetUp() { ints = Enumerable.Range(1, 1000).ToArray(); disposables = new CompositeDisposable(); partitionedStream = Stream .Partitioned <int, int, int>( query: async(q, p) => { return(ints .Where(i => i.IsWithinPartition(p)) .Skip(q.Cursor.Position) .Take(q.BatchSize.Value)); }, advanceCursor: (query, batch) => { // putting the cursor and the partition on the same field is a little weird because a batch of zero doesn't necessarily signify the end of the batch if (batch.Any()) { query.Cursor.AdvanceTo(batch.Last()); } }); Formatter.ListExpansionLimit = 100; Formatter <Projection <HashSet <int>, int> > .RegisterForAllMembers(); }
public static IPartitionedStream <TData, TCursor, TPartition> Trace <TData, TCursor, TPartition>( this IPartitionedStream <TData, TCursor, TPartition> stream) { return(new AnonymousPartitionedStream <TData, TCursor, TPartition>( stream.Id, async p => (await stream.GetStream(p)).Trace())); }
public void SetUp() { ints = Enumerable.Range(1, 1000).ToArray(); disposables = new CompositeDisposable(); partitionedStream = Stream .Partitioned<int, int, int>( query: async (q, p) => { return ints .Where(i => i.IsWithinPartition(p)) .Skip(q.Cursor.Position) .Take(q.BatchSize.Value); }, advanceCursor: (query, batch) => { // putting the cursor and the partition on the same field is a little weird because a batch of zero doesn't necessarily signify the end of the batch if (batch.Any()) { query.Cursor.AdvanceTo(batch.Last()); } }); Formatter.ListExpansionLimit = 100; Formatter<Projection<HashSet<int>, int>>.RegisterForAllMembers(); }
/// <summary> /// Maps data from a stream into a new form. /// </summary> public static IPartitionedStream <TTo, TCursor, TPartition> Map <TFrom, TTo, TCursor, TPartition>( this IPartitionedStream <TFrom, TCursor, TPartition> source, Func <IEnumerable <TFrom>, IEnumerable <TTo> > map, string id = null) { return(new AnonymousPartitionedStream <TTo, TCursor, TPartition>( id: id, getStream: async partition => { var stream = await source.GetStream(partition); return stream.Map(map); })); }
/// <summary> /// Distributes a stream catchup the among one or more partitions using a specified distributor. /// </summary> /// <remarks>If no distributor is provided, then distribution is done in-process.</remarks> public static IStreamCatchup <TData, TCursor> DistributeAmong <TData, TCursor, TPartition>( this IPartitionedStream <TData, TCursor, TPartition> streams, IEnumerable <IStreamQueryPartition <TPartition> > partitions, int?batchSize = null, FetchAndSaveProjection <ICursor <TCursor> > cursorPerPartition = null, IDistributor <IStreamQueryPartition <TPartition> > distributor = null) { return(new DistributedStreamCatchup <TData, TCursor, TPartition>( streams, partitions, batchSize, cursorPerPartition, distributor)); }
public DistributedStreamCatchup( IPartitionedStream <TData, TCursor, TPartition> partitionedStream, IEnumerable <IStreamQueryPartition <TPartition> > partitions, int?batchSize = null, FetchAndSaveProjection <ICursor <TCursor> > manageCursor = null, IDistributor <IStreamQueryPartition <TPartition> > distributor = null) : base(batchSize) { if (partitionedStream == null) { throw new ArgumentNullException("partitionedStream"); } if (partitions == null) { throw new ArgumentNullException("partitions"); } this.partitionedStream = partitionedStream; this.distributor = distributor ?? partitions.DistributeQueriesInProcess(); manageCursor = manageCursor ?? ((id, aggregate) => aggregate(null)); this.distributor #if DEBUG .Trace() // TODO: (DistributedStreamCatchup) figure out a way to let people Trace this distributor #endif .OnReceive(async lease => { await manageCursor(lease.ResourceName, async cursor => { var catchup = new SingleStreamCatchup <TData, TCursor>( await partitionedStream.GetStream(lease.Resource), initialCursor: cursor, batchSize: batchSize, subscriptions: new ConcurrentDictionary <Type, IAggregatorSubscription>(aggregatorSubscriptions)); return(await catchup.RunSingleBatch()); }); }); }
public void SetUp() { words = Values.AtoZ().SelectMany(c => Enumerable.Range(1, 100).Select(i => c + i)).ToList(); partitions = Values.AtoZ().Select(c => Partition.Where <string>(s => s.StartsWith(c), named: c)); partitionedStream = Stream .Partitioned <string, int, string>( query: async(q, partition) => { var wordsInPartition = words .Skip(q.Cursor.Position) .Where(partition.Contains); var b = wordsInPartition .Take(q.BatchSize.Value); return(b); }, advanceCursor: (query, batch) => { query.Cursor.AdvanceTo(words.IndexOf(batch.Last()) + 1); }); Formatter.ListExpansionLimit = 100; Formatter <Projection <HashSet <int>, int> > .RegisterForAllMembers(); }
public void SetUp() { words = Values.AtoZ().SelectMany(c => Enumerable.Range(1, 100).Select(i => c + i)).ToList(); partitions = Values.AtoZ().Select(c => Partition.Where<string>(s => s.StartsWith(c), named: c)); partitionedStream = Stream .Partitioned<string, int, string>( query: async (q, partition) => { var wordsInPartition = words .Skip(q.Cursor.Position) .Where(partition.Contains); var b = wordsInPartition .Take(q.BatchSize.Value); return b; }, advanceCursor: (query, batch) => { query.Cursor.AdvanceTo(words.IndexOf(batch.Last()) + 1); }); Formatter.ListExpansionLimit = 100; Formatter<Projection<HashSet<int>, int>>.RegisterForAllMembers(); }