public static Stream <R, T> Broadcast <R, T>(this Stream <R, T> stream)
where R : Cloneable <R>
where T : Time <T>
{
var controller = stream.ForStage.InternalGraphManager.Controller;
int threadCount = stream.ForStage.InternalGraphManager.DefaultPlacement.Count / controller.Configuration.Processes;
if (threadCount * controller.Configuration.Processes != stream.ForStage.InternalGraphManager.DefaultPlacement.Count)
{
throw new Exception("Uneven thread count?");
}
var processDests = stream.ForStage.InternalGraphManager.DefaultPlacement.Where(x => x.ThreadId == 0).Select(x => x.VertexId).ToArray();
var boutput = UnaryVertex <R, Pair <int, R>, T> .MakeStage(stream, (i, v) => new BroadcastSendShard <R, T>(i, v, processDests), null, null, "BroadcastProcessSend");
var collectable = boutput;
if (stream.ForStage.InternalGraphManager.DefaultPlacement.Where(x => x.ProcessId == controller.Configuration.ProcessID).Count() > 1)
{
var threadDests = stream.ForStage.InternalGraphManager.DefaultPlacement
.Where(x => x.ProcessId == controller.Configuration.ProcessID)
.Select(x => x.VertexId)
.ToArray();
collectable = UnaryVertex <Pair <int, R>, Pair <int, R>, T> .MakeStage(boutput, (i, v) => new BroadcastForwardShard <R, T>(i, v, threadDests), x => x.v1, null, "BroadcastShardSend");
}
// TODO : fix this to use a streaming expression
return(collectable.UnaryExpression(null, xs => xs.Select(x => x.v2), "Select"));
}
public static Stream <Pair <K, X>, T> LocalTimeReduce <A, X, R, S, K, I, T>(
this Stream <I, T> stream, Func <I, K> key, Func <I, R> val, Func <A> factory, string name,
Expression <Func <I, int> > inPlacement, Expression <Func <Pair <K, X>, int> > outPlacement)
where A : IReducer <X, R, S>
where T : Time <T>
{
return(UnaryVertex <I, Pair <K, X>, T> .MakeStage(stream, (i, v) => new LocalTimeKeyedReduceShard <A, X, R, S, K, I, T>(i, v, key, val, factory), inPlacement, outPlacement, name));
}
public static Stream <Pair <TKey, TState>, TTime> LocalReduce <TReducer, TState, TValue, TOutput, TKey, TInput, TTime>(
this Stream <TInput, TTime> stream, Func <TInput, TKey> key, Func <TInput, TValue> val, Func <TReducer> factory, string name,
Expression <Func <TInput, int> > inPlacement, Expression <Func <Pair <TKey, TState>, int> > outPlacement)
where TReducer : IReducer <TState, TValue, TOutput>
where TTime : Time <TTime>
{
return(UnaryVertex <TInput, Pair <TKey, TState>, TTime> .MakeStage(stream, (i, v) => new LocalKeyedReduceShard <TReducer, TState, TValue, TOutput, TKey, TInput, TTime>(i, v, key, val, factory), inPlacement, outPlacement, name));
}
public static Stream <Pair <K, S>, T> LocalTimeCombine <A, X, R, S, K, T>(
this Stream <Pair <K, X>, T> stream, Func <A> factory, string name,
Expression <Func <Pair <K, S>, int> > outPlacement)
where A : IReducer <X, R, S>
where T : Time <T>
{
Expression <Func <Pair <K, X>, int> > inPlacement = null;
if (outPlacement != null)
{
inPlacement = x => x.v1.GetHashCode();
}
return(UnaryVertex <Pair <K, X>, Pair <K, S>, T> .MakeStage(stream, (i, v) => new LocalTimeKeyedCombineShard <A, X, R, S, K, T>(i, v, factory), inPlacement, outPlacement, name));
}
/// <summary>
/// One to many record by record transformation
/// </summary>
/// <typeparam name="TInput">Input type</typeparam>
/// <typeparam name="TOutput">Output type</typeparam>
/// <typeparam name="TTime">Time type</typeparam>
/// <param name="stream">input stream</param>
/// <param name="function">transformation</param>
/// <returns>the concatenation of all results produced from each input record</returns>
public static Stream <TOutput, TTime> SelectMany <TInput, TOutput, TTime>(this Stream <TInput, TTime> stream, Func <TInput, IEnumerable <TOutput> > function) where TTime : Time <TTime>
{
return(UnaryVertex <TInput, TOutput, TTime> .MakeStage(stream, (i, v) => new SelectManyVertex <TInput, TOutput, TTime>(i, v, function), null, null, "SelectMany"));
}
/// <summary>
/// Select with access to shard index information
/// </summary>
/// <typeparam name="TInput">Input type</typeparam>
/// <typeparam name="TOutput">Output type</typeparam>
/// <typeparam name="TTime">Time type</typeparam>
/// <param name="stream">input stream</param>
/// <param name="function">transformation</param>
/// <returns>record by record transformation of the input stream</returns>
public static Stream <TOutput, TTime> SelectShard <TInput, TOutput, TTime>(this Stream <TInput, TTime> stream, Func <int, TInput, TOutput> function)
where TTime : Time <TTime>
{
return(UnaryVertex <TInput, TOutput, TTime> .MakeStage(stream, (i, v) => new ShardSelect <TInput, TOutput, TTime>(i, v, function), null, null, "ShardSelect"));
}
/// <summary>
/// Record by record filtering
/// </summary>
/// <typeparam name="TRecord">Input type</typeparam>
/// <typeparam name="TTime">Time type</typeparam>
/// <param name="stream">input stream</param>
/// <param name="predicate">predicate</param>
/// <returns>filtered stream</returns>
public static Stream <TRecord, TTime> Where <TRecord, TTime>(this Stream <TRecord, TTime> stream, Func <TRecord, bool> predicate) where TTime : Time <TTime>
{
return(UnaryVertex <TRecord, TRecord, TTime> .MakeStage(stream, (i, v) => new WhereVertex <TRecord, TTime>(i, v, predicate), stream.PartitionedBy, stream.PartitionedBy, "Where"));
}
public static Stream <TOutput, TTime> LocalCombine <TReducer, TState, TInput, TOutput, TTime>(this Stream <TState, TTime> stream, Func <TReducer> factory, string name)
where TReducer : IReducer <TState, TInput, TOutput>
where TTime : Time <TTime>
{
return(UnaryVertex <TState, TOutput, TTime> .MakeStage(stream, (i, v) => new LocalCombineShard <TReducer, TState, TInput, TOutput, TTime>(i, v, factory), null, null, name));
}
