internal static Stream <R, T> Broadcast <R, T>(this Stream <R, T> stream)
where R : Cloneable <R>
where T : Time <T>
{
var controller = stream.ForStage.InternalComputation.Controller;
int threadCount = stream.ForStage.InternalComputation.DefaultPlacement.Count / controller.Configuration.Processes;
if (threadCount * controller.Configuration.Processes != stream.ForStage.InternalComputation.DefaultPlacement.Count)
{
throw new Exception("Uneven thread count?");
}
var processDests = stream.ForStage.InternalComputation.DefaultPlacement.Where(x => x.ThreadId == 0).Select(x => x.VertexId).ToArray();
var boutput = Foundry.NewUnaryStage(stream, (i, v) => new BroadcastSendVertex <R, T>(i, v, processDests), null, null, "BroadcastProcessSend");
var collectable = boutput;
if (stream.ForStage.InternalComputation.DefaultPlacement.Where(x => x.ProcessId == controller.Configuration.ProcessID).Count() > 1)
{
var threadDests = stream.ForStage.InternalComputation.DefaultPlacement
.Where(x => x.ProcessId == controller.Configuration.ProcessID)
.Select(x => x.VertexId)
.ToArray();
collectable = Foundry.NewUnaryStage(boutput, (i, v) => new BroadcastForwardVertex <R, T>(i, v, threadDests), x => x.First, null, "BroadcastVertexSend");
}
// TODO : fix this to use a streaming expression
return(collectable.UnaryExpression(null, xs => xs.Select(x => x.Second), "Select"));
}
private InternalCollection <R2, T> Manufacture <R2>(Func <int, Stage <T>, UnaryVertex <Weighted <R>, Weighted <R2>, T> > factory, Expression <Func <Weighted <R>, int> > inputPartitionedBy, Expression <Func <Weighted <R2>, int> > outputPartitionedBy, string name)
where R2 : IEquatable <R2>
{
var output = Foundry.NewUnaryStage(this.Output, factory, inputPartitionedBy, outputPartitionedBy, name);
return(new InternalCollection <R2, T>(output, this.Immutable));
}
internal static Stream <Pair <K, X>, T> LocalTimeReduce <A, X, R, S, K, I, T>(
this Stream <I, T> stream, Func <I, K> key, Func <I, R> val, Func <A> factory, string name,
Expression <Func <I, int> > inPlacement, Expression <Func <Pair <K, X>, int> > outPlacement)
where A : IReducer <X, R, S>
where T : Time <T>
{
return(Foundry.NewUnaryStage(stream, (i, v) => new LocalTimeKeyedReduceVertex <A, X, R, S, K, I, T>(i, v, key, val, factory), inPlacement, outPlacement, name));
}
internal static Stream <Pair <TKey, TState>, TTime> LocalReduce <TReducer, TState, TValue, TOutput, TKey, TInput, TTime>(
this Stream <TInput, TTime> stream, Func <TInput, TKey> key, Func <TInput, TValue> val, Func <TReducer> factory, string name,
Expression <Func <TInput, int> > inPlacement, Expression <Func <Pair <TKey, TState>, int> > outPlacement)
where TReducer : IReducer <TState, TValue, TOutput>
where TTime : Time <TTime>
{
return(Foundry.NewUnaryStage(stream, (i, v) => new LocalKeyedReduceVertex <TReducer, TState, TValue, TOutput, TKey, TInput, TTime>(i, v, key, val, factory), inPlacement, outPlacement, name));
}
internal static Stream <Pair <K, S>, T> LocalTimeCombine <A, X, R, S, K, T>(
this Stream <Pair <K, X>, T> stream, Func <A> factory, string name,
Expression <Func <Pair <K, S>, int> > outPlacement)
where A : IReducer <X, R, S>
where T : Time <T>
{
Expression <Func <Pair <K, X>, int> > inPlacement = null;
if (outPlacement != null)
{
inPlacement = x => x.First.GetHashCode();
}
return(Foundry.NewUnaryStage(stream, (i, v) => new LocalTimeKeyedCombineVertex <A, X, R, S, K, T>(i, v, factory), inPlacement, outPlacement, name));
}
/// <summary>
/// Aggregates key-value pairs, producing new outputs whenever an aggregate changes.
/// </summary>
/// <typeparam name="TKey">key type</typeparam>
/// <typeparam name="TValue">value type</typeparam>
/// <typeparam name="TTime">time type</typeparam>
/// <param name="input">input key-value stream</param>
/// <param name="aggregate">aggregation function</param>
/// <returns>aggregated key-value pairs</returns>
public static Stream <Pair <TKey, TValue>, TTime> StreamingAggregate <TKey, TValue, TTime>(this Stream <Pair <TKey, TValue>, TTime> input, Func <TValue, TValue, TValue> aggregate)
where TTime : Time <TTime>
where TValue : IEquatable <TValue>
{
return(Foundry.NewUnaryStage(input, (i, s) => new StreamingAggregateVertex <TKey, TValue, TTime>(i, s, aggregate), x => x.First.GetHashCode(), x => x.First.GetHashCode(), "StreamingAggregate"));
}
internal static Stream <TOutput, TTime> LocalCombine <TReducer, TState, TInput, TOutput, TTime>(this Stream <TState, TTime> stream, Func <TReducer> factory, string name)
where TReducer : IReducer <TState, TInput, TOutput>
where TTime : Time <TTime>
{
return(Foundry.NewUnaryStage(stream, (i, v) => new LocalCombineVertex <TReducer, TState, TInput, TOutput, TTime>(i, v, factory), null, null, name));
}