/// <summary>
/// Shuffle operation on a sharded streamable
/// </summary>
/// <typeparam name="TKey">Grouping/sharding key type for input data</typeparam>
/// <typeparam name="TNewKey">Grouping/sharding key type for output data</typeparam>
/// <typeparam name="TPayload">Payload type for data flowing through the shuffle operation</typeparam>
/// <param name="streamable">Input sharded streamable for the shuffle operation</param>
/// <param name="shuffleSelector">Selector function to determine new shard keys</param>
/// <param name="newLocation">Assign an optional new location descriptor</param>
/// <returns>A new sharded streamable post-shuffle</returns>
public static IShardedStreamable <TNewKey, TPayload> Shuffle <TKey, TNewKey, TPayload>(
this IShardedStreamable <TKey, TPayload> streamable,
Expression <Func <TPayload, TNewKey> > shuffleSelector,
ILocationDescriptor newLocation = null)
{
return(streamable.ReKey(shuffleSelector).ReDistribute(newLocation));
}
/// <summary>
/// Break up a sharded streamable into an array of sharded streamables, each consisting of only one shard
/// </summary>
/// <returns>Array of sharded streamables</returns>
public IShardedStreamable <TKey, TPayload>[] Split()
{
var result = new IShardedStreamable <TKey, TPayload> [this.Streamables.Length];
for (int i = 0; i < result.Length; i++)
{
var singletonArray = new IStreamable <TKey, TPayload> [1];
singletonArray[0] = this.Streamables[i];
result[i] = new ShardedStreamable <TKey, TPayload>(singletonArray);
}
return(result);
}
/// <summary>
/// Execute a binary query on all shards
/// </summary>
/// <typeparam name="TPayload2">Event payload type for data from the second input</typeparam>
/// <typeparam name="TOutput">The type of event payload in the output</typeparam>
/// <param name="input2">The second input to the binary query</param>
/// <param name="query">The query to evaluate</param>
/// <returns>A new sharded streamable comprised of the output of the query</returns>
public IShardedStreamable <TKey, TOutput> Query <TPayload2, TOutput>(
IShardedStreamable <TKey, TPayload2> input2,
Expression <Func <IStreamable <TKey, TPayload>, IStreamable <TKey, TPayload2>, IStreamable <TKey, TOutput> > > query)
{
var result = new IStreamable <TKey, TOutput> [this.streamables.Length];
var queryC = query.Compile();
for (int i = 0; i < this.streamables.Length; i++)
{
result[i] = queryC(this.streamables[i], ((ShardedStreamable <TKey, TPayload2>)input2).streamables[i]);
}
return(new ShardedStreamable <TKey, TOutput>(result));
}
/// <summary>
/// Write stream properties to specified .NET stream
/// </summary>
/// <typeparam name="TKey"></typeparam>
/// <typeparam name="TPayload"></typeparam>
/// <param name="streamable"></param>
/// <param name="stream"></param>
public static void WritePropertiesToStream <TKey, TPayload>(this IShardedStreamable <TKey, TPayload> streamable, Stream stream)
{
var properties = StreamProperties <TKey, TPayload> .Default;
var shardedStreamable = (ShardedStreamable <TKey, TPayload>)streamable;
if (shardedStreamable != null)
{
if (shardedStreamable.Streamables.Length > 0)
{
properties = shardedStreamable.Streamables[0].Properties;
}
}
var propSer = StreamSerializer.Create <SerializedProperties>();
propSer.Serialize(stream, SerializedProperties.FromStreamProperties(properties));
}
/// <summary>
/// Constructor to take a sharded streamable and provide serialization features
/// </summary>
/// <param name="source">The sharded streamable from which data shall be serialized</param>
/// <param name="destinations">The sinks to which serialized data should be written</param>
/// <param name="aSync">States whether serialization should be able to be done asynchronously</param>
/// <param name="writePropertiesToStream">States whether streams properties should be written to the binary stream</param>
public ShardedStreamSerializer(IShardedStreamable <TKey, TPayload> source, Stream[] destinations, bool aSync = false, bool writePropertiesToStream = false)
{
var shardedSource = (ShardedStreamable <TKey, TPayload>)source;
var observers = new ShardedSerializerObserver <TKey, TPayload> [shardedSource.Streamables.Length];
for (int i = 0; i < shardedSource.Streamables.Length; i++)
{
observers[i] = new ShardedSerializerObserver <TKey, TPayload>(destinations[i], shardedSource.Streamables[i].Properties, writePropertiesToStream);
shardedSource.Streamables[i].ToStreamMessageObservable().Subscribe(observers[i]);
}
if (!aSync)
{
for (int i = 0; i < observers.Length; i++)
{
observers[i].Wait();
}
}
return;
}
/// <summary>
/// Create a cached sharded streamable from an existing sharded streamable
/// </summary>
/// <param name="source">The sharded streamable to cache</param>
public ShardedStreamCache(IShardedStreamable <TKey, TPayload> source)
{
var shardSource = (ShardedStreamable <TKey, TPayload>)source;
this.streamables = new IStreamable <TKey, TPayload> [shardSource.Streamables.Length];
this.caches = new StreamCache <TKey, TPayload> [shardSource.Streamables.Length];
var observers = new ShardedCacheObserver <TKey, TPayload> [shardSource.Streamables.Length];
for (int i = 0; i < shardSource.Streamables.Length; i++)
{
this.caches[i] = new StreamCache <TKey, TPayload>();
observers[i] = new ShardedCacheObserver <TKey, TPayload>(this.caches[i], shardSource.Streamables[i].Properties);
this.streamables[i] = this.caches[i];
shardSource.Streamables[i].ToStreamMessageObservable().Subscribe(observers[i]);
}
for (int i = 0; i < shardSource.Streamables.Length; i++)
{
observers[i].Wait();
}
return;
}
/// <summary>
/// Unshuffle operation on a sharded streamable (essentially, a unifying shuffle operation shuffling to a unity shard
/// </summary>
/// <typeparam name="TKey">Grouping/sharding key type for input data</typeparam>
/// <typeparam name="TPayload">Payload type for data flowing through the shuffle operation</typeparam>
/// <param name="streamable">Input sharded streamable for the shuffle operation</param>
/// <returns>A new sharded streamable post-shuffle</returns>
public static IShardedStreamable <Empty, TPayload> Unshuffle <TKey, TPayload>(this IShardedStreamable <TKey, TPayload> streamable)
{
return(streamable.ReKey(e => Empty.Default));
}
/// <summary> /// Create a cache from a sharded streamable /// </summary> /// <typeparam name="TKey">Grouping key type for data in the query</typeparam> /// <typeparam name="TPayload">Event payload type for data in the query</typeparam> /// <param name="source">The sharded streamable to cache</param> /// <returns>A cached sharded streamable</returns> public static ShardedStreamCache <TKey, TPayload> Cache <TKey, TPayload>(this IShardedStreamable <TKey, TPayload> source) => new ShardedStreamCache <TKey, TPayload>(source);
/// <summary> /// Unshard operation on a partitioned stream /// </summary> /// <typeparam name="TKey">The partition key type</typeparam> /// <typeparam name="TPayload">The event payload type</typeparam> /// <param name="source">The stream to be returned from sharded to strictly a unified stream</param> /// <returns>A streamable brought together from all shards</returns> public static IStreamable <PartitionKey <TKey>, TPayload> Unshard <TKey, TPayload>(this IShardedStreamable <PartitionKey <TKey>, TPayload> source) => new MultiUnionStreamable <PartitionKey <TKey>, TPayload>(((ShardedStreamable <PartitionKey <TKey>, TPayload>)source).Streamables, false);
/// <summary> /// Unshard operation on a non-partitioned stream /// </summary> /// <typeparam name="TPayload">The event payload type</typeparam> /// <param name="source">The stream to be returned from sharded to strictly a unified stream</param> /// <returns>A streamable brought together from all shards</returns> public static IStreamable <Empty, TPayload> Unshard <TPayload>(this IShardedStreamable <Empty, TPayload> source) => new MultiUnionStreamable <Empty, TPayload>(((ShardedStreamable <Empty, TPayload>)source).Streamables, false);
/// <summary> /// Stream data from a sharded streamable to binary streams /// </summary> /// <typeparam name="TKey">Grouping key type for data in the query</typeparam> /// <typeparam name="TPayload">Event payload type for data in the query</typeparam> /// <param name="source">The sharded streamable from which data shall be serialized</param> /// <param name="destinations">The sinks to which serialized data should be written</param> /// <param name="async">States whether serialization should be able to be done asynchronously</param> /// <param name="writePropertiesToStream">Write stream properties to the binary stream</param> public static void ToBinaryStream <TKey, TPayload>(this IShardedStreamable <TKey, TPayload> source, Stream[] destinations, bool async = false, bool writePropertiesToStream = false) => new ShardedStreamSerializer <TKey, TPayload>(source, destinations, async, writePropertiesToStream);
/// <summary>
/// Create a cache from a sharded streamable
/// </summary>
/// <typeparam name="TKey">Grouping key type for data in the query</typeparam>
/// <typeparam name="TPayload">Event payload type for data in the query</typeparam>
/// <param name="source">The sharded streamable to cache</param>
/// <returns>A cached sharded streamable</returns>
public static ShardedStreamCache <TKey, TPayload> Cache <TKey, TPayload>(this IShardedStreamable <TKey, TPayload> source)
{
return(new ShardedStreamCache <TKey, TPayload>(source));
}
