/// <summary>
/// Shuffle operation on a sharded streamable
/// </summary>
/// <typeparam name="TKey">Grouping/sharding key type for input data</typeparam>
/// <typeparam name="TNewKey">Grouping/sharding key type for output data</typeparam>
/// <typeparam name="TPayload">Payload type for data flowing through the shuffle operation</typeparam>
/// <param name="streamable">Input sharded streamable for the shuffle operation</param>
/// <param name="shuffleSelector">Selector function to determine new shard keys</param>
/// <param name="newLocation">Assign an optional new location descriptor</param>
/// <returns>A new sharded streamable post-shuffle</returns>
public static IShardedStreamable <TNewKey, TPayload> Shuffle <TKey, TNewKey, TPayload>(
this IShardedStreamable <TKey, TPayload> streamable,
Expression <Func <TPayload, TNewKey> > shuffleSelector,
ILocationDescriptor newLocation = null)
{
return(streamable.ReKey(shuffleSelector).ReDistribute(newLocation));
}
/// <summary>
/// Re-distribute the data across shards
/// </summary>
/// <param name="newLocation">Assign an optional new location descriptor</param>
/// <returns>A new sharded streamable with the new key type and values</returns>
public IShardedStreamable <TKey, TPayload> ReDistribute(ILocationDescriptor newLocation = null)
{
int newShardArity = (newLocation != null && (int)newLocation.GetLocation() > 0) ? (int)newLocation.GetLocation() : this.Streamables.Length;
var shuffleL1Results = new ShuffleSameKeyStreamable <TKey, TPayload, TKey> [this.Streamables.Length];
for (int i = 0; i < this.Streamables.Length; i++)
{
shuffleL1Results[i] = new ShuffleSameKeyStreamable <TKey, TPayload, TKey>(this.Streamables[i], newShardArity, i);
}
var shuffleResults = new IStreamable <TKey, TPayload> [newShardArity];
for (int i = 0; i < newShardArity; i++)
{
shuffleResults[i] = new MultiUnionStreamable <TKey, TPayload>(shuffleL1Results);
}
return(new ShardedStreamable <TKey, TPayload>(shuffleResults));
}
/// <summary>
/// Broadcast operation on a sharded streamable
/// </summary>
/// <param name="newLocation">Assign an optional new location descriptor</param>
/// <returns>A new sharded streamable post-broadcast</returns>
public IShardedStreamable <TKey, TPayload> Broadcast(ILocationDescriptor newLocation = null)
{
int newShardArity = (newLocation != null && (int)newLocation.GetLocation() > 0) ? (int)newLocation.GetLocation() : this.Streamables.Length;
var broadcastResults = new SprayGroupImportStreamable <TKey, TPayload> [this.Streamables.Length];
for (int i = 0; i < this.Streamables.Length; i++)
{
broadcastResults[i] = new SprayGroupImportStreamable <TKey, TPayload>(this.Streamables[i], newShardArity, true);
}
var unionResults = new IStreamable <TKey, TPayload> [newShardArity];
for (int i = 0; i < newShardArity; i++)
{
unionResults[i] = new MultiUnionStreamable <TKey, TPayload>(broadcastResults);
}
return(new ShardedStreamable <TKey, TPayload>(unionResults));
}
/// <summary>
/// Multicast operation on a sharded streamable
/// </summary>
/// <param name="destination">A destination desciption for the multicast operation</param>
/// <param name="newLocation">Assign an optional new location descriptor</param>
/// <returns>A new sharded streamable post-multicast</returns>
public IShardedStreamable <TKey, TPayload> Multicast(IDestinationDescriptor destination, ILocationDescriptor newLocation = null)
{
int newShardArity = (newLocation != null && (int)newLocation.GetLocation() > 0) ? (int)newLocation.GetLocation() : this.Streamables.Length;
var destinationSelector = (Expression <Func <TKey, int, TPayload, int[]> >)destination.GetDestination();
var shufflecastL1Results = new ShufflecastStreamable <TPayload, TKey> [this.Streamables.Length];
for (int i = 0; i < this.Streamables.Length; i++)
{
shufflecastL1Results[i] = new ShufflecastStreamable <TPayload, TKey>(null, this.Streamables[i], newShardArity, destinationSelector);
}
var shuffleResults = new IStreamable <TKey, TPayload> [newShardArity];
for (int i = 0; i < newShardArity; i++)
{
shuffleResults[i] = new MultiUnionStreamable <TKey, TPayload>(shufflecastL1Results);
}
return(new ShardedStreamable <TKey, TPayload>(shuffleResults));
}
