/// <summary> /// Shuffle operation on a sharded streamable /// </summary> /// <typeparam name="TKey">Grouping/sharding key type for input data</typeparam> /// <typeparam name="TNewKey">Grouping/sharding key type for output data</typeparam> /// <typeparam name="TPayload">Payload type for data flowing through the shuffle operation</typeparam> /// <param name="streamable">Input sharded streamable for the shuffle operation</param> /// <param name="shuffleSelector">Selector function to determine new shard keys</param> /// <param name="newLocation">Assign an optional new location descriptor</param> /// <returns>A new sharded streamable post-shuffle</returns> public static IShardedStreamable <TNewKey, TPayload> Shuffle <TKey, TNewKey, TPayload>( this IShardedStreamable <TKey, TPayload> streamable, Expression <Func <TPayload, TNewKey> > shuffleSelector, ILocationDescriptor newLocation = null) { return(streamable.ReKey(shuffleSelector).ReDistribute(newLocation)); }
/// <summary> /// Re-distribute the data across shards /// </summary> /// <param name="newLocation">Assign an optional new location descriptor</param> /// <returns>A new sharded streamable with the new key type and values</returns> public IShardedStreamable <TKey, TPayload> ReDistribute(ILocationDescriptor newLocation = null) { int newShardArity = (newLocation != null && (int)newLocation.GetLocation() > 0) ? (int)newLocation.GetLocation() : this.Streamables.Length; var shuffleL1Results = new ShuffleSameKeyStreamable <TKey, TPayload, TKey> [this.Streamables.Length]; for (int i = 0; i < this.Streamables.Length; i++) { shuffleL1Results[i] = new ShuffleSameKeyStreamable <TKey, TPayload, TKey>(this.Streamables[i], newShardArity, i); } var shuffleResults = new IStreamable <TKey, TPayload> [newShardArity]; for (int i = 0; i < newShardArity; i++) { shuffleResults[i] = new MultiUnionStreamable <TKey, TPayload>(shuffleL1Results); } return(new ShardedStreamable <TKey, TPayload>(shuffleResults)); }
/// <summary> /// Broadcast operation on a sharded streamable /// </summary> /// <param name="newLocation">Assign an optional new location descriptor</param> /// <returns>A new sharded streamable post-broadcast</returns> public IShardedStreamable <TKey, TPayload> Broadcast(ILocationDescriptor newLocation = null) { int newShardArity = (newLocation != null && (int)newLocation.GetLocation() > 0) ? (int)newLocation.GetLocation() : this.Streamables.Length; var broadcastResults = new SprayGroupImportStreamable <TKey, TPayload> [this.Streamables.Length]; for (int i = 0; i < this.Streamables.Length; i++) { broadcastResults[i] = new SprayGroupImportStreamable <TKey, TPayload>(this.Streamables[i], newShardArity, true); } var unionResults = new IStreamable <TKey, TPayload> [newShardArity]; for (int i = 0; i < newShardArity; i++) { unionResults[i] = new MultiUnionStreamable <TKey, TPayload>(broadcastResults); } return(new ShardedStreamable <TKey, TPayload>(unionResults)); }
/// <summary> /// Multicast operation on a sharded streamable /// </summary> /// <param name="destination">A destination desciption for the multicast operation</param> /// <param name="newLocation">Assign an optional new location descriptor</param> /// <returns>A new sharded streamable post-multicast</returns> public IShardedStreamable <TKey, TPayload> Multicast(IDestinationDescriptor destination, ILocationDescriptor newLocation = null) { int newShardArity = (newLocation != null && (int)newLocation.GetLocation() > 0) ? (int)newLocation.GetLocation() : this.Streamables.Length; var destinationSelector = (Expression <Func <TKey, int, TPayload, int[]> >)destination.GetDestination(); var shufflecastL1Results = new ShufflecastStreamable <TPayload, TKey> [this.Streamables.Length]; for (int i = 0; i < this.Streamables.Length; i++) { shufflecastL1Results[i] = new ShufflecastStreamable <TPayload, TKey>(null, this.Streamables[i], newShardArity, destinationSelector); } var shuffleResults = new IStreamable <TKey, TPayload> [newShardArity]; for (int i = 0; i < newShardArity; i++) { shuffleResults[i] = new MultiUnionStreamable <TKey, TPayload>(shufflecastL1Results); } return(new ShardedStreamable <TKey, TPayload>(shuffleResults)); }