/// <summary>
/// Applies the given window function to each window. The window function is called for each evaluation of the window for each key individually. The output of the window function is interpreted as a regular non-windowed stream.
/// Arriving data is incrementally aggregated using the given reducer.
/// </summary>
/// <typeparam name="TOutput"></typeparam>
/// <param name="reduceFunction">The reduce function that is used for incremental aggregation.</param>
/// <param name="function">The window function.</param>
/// <returns>The data stream that is the result of applying the window function to the window.</returns>
public SingleOutputStreamOperator <TOutput> Reduce <TOutput>(IReduceFunction <TElement> reduceFunction, IWindowFunction <TElement, TOutput, TKey, TWindow> function)
{
var inType = _input.Type;
var resultType = getWindowFunctionReturnType(function, inType);
return(Reduce(reduceFunction, function, resultType));
}
private ReduceReceiver(
[Parameter(typeof(GroupCommConfigurationOptions.OperatorName))] string operatorName,
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.Initialize))] bool initialize,
OperatorTopology <PipelineMessage <T> > topology,
ICommunicationGroupNetworkObserver networkHandler,
IReduceFunction <T> reduceFunction,
IPipelineDataConverter <T> dataConverter)
{
OperatorName = operatorName;
GroupName = groupName;
Version = PipelineVersion;
ReduceFunction = reduceFunction;
PipelineDataConverter = dataConverter;
_pipelinedReduceFunc = new PipelinedReduceFunction <T>(ReduceFunction);
_topology = topology;
var msgHandler = Observer.Create <GeneralGroupCommunicationMessage>(message => topology.OnNext(message));
networkHandler.Register(operatorName, msgHandler);
if (initialize)
{
topology.Initialize();
}
}
public ReducingStateDescriptor(string name,
IReduceFunction <TValue> reduceFunction,
System.Type type,
TValue defaultValue = default)
: base(name, type, defaultValue)
{
}
/// <summary>
/// Applies the given window function to each window. The window function is called for each evaluation of the window for each key individually. The output of the window function is interpreted as a regular non-windowed stream.
/// Arriving data is incrementally aggregated using the given reducer.
/// </summary>
/// <typeparam name="TOutput"></typeparam>
/// <param name="reduceFunction">The reduce function that is used for incremental aggregation.</param>
/// <param name="function">The window function.</param>
/// <param name="resultType">Type information for the result type of the window function.</param>
/// <returns>The data stream that is the result of applying the window function to the window.</returns>
public SingleOutputStreamOperator <TOutput> Reduce <TOutput>(
IReduceFunction <TElement> reduceFunction,
IWindowFunction <TElement, TOutput, TKey, TWindow> function,
TypeInformation <TOutput> resultType)
{
if (reduceFunction is IRichFunction)
{
throw new InvalidOperationException("ReduceFunction of reduce can not be a RichFunction.");
}
var env = _input.ExecutionEnvironment;
var config = env.ExecutionConfig;
function = env.Clean(function);
reduceFunction = env.Clean(reduceFunction);
var operatorName = GenerateOperatorName(_assigner, _trigger, _evictor, reduceFunction, function);
var keySelector = _input.KeySelector;
IOneInputStreamOperator <TElement, TOutput> @operator = default;
if (_evictor != null)
{
}
else
{
var stateDesc = new ReducingStateDescriptor <TElement>("window-contents", reduceFunction, _input.Type.CreateSerializer(config));
}
return(_input.Transform(operatorName, resultType, @operator));
}
/// <summary>
/// Receives all messages from child Tasks and reduces them with the
/// given IReduceFunction.
/// </summary>
/// <param name="reduceFunction">The class used to reduce messages</param>
/// <returns>The result of reducing messages</returns>
public T ReceiveFromChildren(IReduceFunction <T> reduceFunction)
{
if (reduceFunction == null)
{
throw new ArgumentNullException("reduceFunction");
}
var receivedData = new List <T>();
var childrenToReceiveFrom = new HashSet <string>(_children.Select(node => node.Identifier));
while (childrenToReceiveFrom.Count > 0)
{
var childrenWithData = GetNodeWithData(childrenToReceiveFrom);
foreach (var child in childrenWithData)
{
T[] data = ReceiveFromNode(child);
if (data == null || data.Length != 1)
{
throw new InvalidOperationException("Received invalid data from child with id: " + child.Identifier);
}
receivedData.Add(data[0]);
childrenToReceiveFrom.Remove(child.Identifier);
}
}
return(reduceFunction.Reduce(receivedData));
}
public ReducingStateDescriptor(string name,
IReduceFunction <TValue> reduceFunction,
TypeInformation <TValue> typeInfo,
TValue defaultValue = default)
: base(name, typeInfo, defaultValue)
{
ReduceFunction = reduceFunction;
}
public ReducingStateDescriptor(string name,
IReduceFunction <TValue> reduceFunction,
TypeSerializer <TValue> typeSerializer,
TValue defaultValue = default)
: base(name, typeSerializer, defaultValue)
{
ReduceFunction = reduceFunction;
}
private IMRURunner(MapFunctions <TMapInput, TMapOutput> mapfunctions,
IReduceFunction <TMapOutput> reduceTask,
IUpdateFunction <TMapInput, TMapOutput, TResult> updateTask)
{
_mapfunctions = mapfunctions.Mappers;
_reduceTask = reduceTask;
_updateTask = updateTask;
}
/// <summary>
/// Receives all messages from child Tasks and reduces them with the
/// given IReduceFunction.
/// </summary>
/// <param name="reduceFunction">The class used to reduce messages</param>
/// <returns>The result of reducing messages</returns>
public T ReceiveFromChildren(IReduceFunction <T> reduceFunction)
{
if (reduceFunction == null)
{
throw new ArgumentNullException("reduceFunction");
}
return(reduceFunction.Reduce(_childNodeContainer.GetDataFromAllChildren()));
}
/// <summary>
/// Receives all messages from child Tasks and reduces them with the
/// given IReduceFunction.
/// </summary>
/// <param name="reduceFunction">The class used to reduce messages</param>
/// <param name="cancellationSource">The cancellation token for the data reading operation cancellation</param>
/// <returns>The result of reducing messages</returns>
public T ReceiveFromChildren(IReduceFunction <T> reduceFunction, CancellationTokenSource cancellationSource = null)
{
if (reduceFunction == null)
{
throw new ArgumentNullException("reduceFunction");
}
return(reduceFunction.Reduce(_childNodeContainer.GetDataFromAllChildren(cancellationSource)));
}
/// <summary>
/// Applies a reduce function to the window. The window function is called for each evaluation of the window for each key individually. The output of the reduce function is interpreted as a regular non-windowed stream.
/// This window will try and incrementally aggregate data as much as the window policies permit. For example, tumbling time windows can aggregate the data, meaning that only one element per key is stored. Sliding time windows will aggregate on the granularity of the slide interval, so a few elements are stored per key (one per slide interval). Custom windows may not be able to incrementally aggregate, or may need to store extra values in an aggregation tree.
/// </summary>
/// <param name="function">The reduce function.</param>
/// <returns>The data stream that is the result of applying the reduce function to the window.</returns>
public SingleOutputStreamOperator <TElement> Reduce(IReduceFunction <TElement> function)
{
if (function is IRichFunction)
{
throw new InvalidOperationException("ReduceFunction of reduce can not be a RichFunction. " +
"Please use reduce(ReduceFunction, WindowFunction) instead.");
}
//clean the closure
function = _input.ExecutionEnvironment.Clean(function);
return(Reduce(function, new PassThroughWindowFunction <TKey, TWindow, TElement>()));
}
public void TestConfigurationReduceSpec()
{
FlatTopology <int> topology = new FlatTopology <int>("Operator", "Group", "task1", "driverid",
new ReduceOperatorSpec("task1", Configurations.Merge(GetDefaultCodecConfig(), GetDefaultDataConverterConfig(), GetDefaultReduceFuncConfig())));
topology.AddTask("task1");
var conf2 = topology.GetTaskConfiguration("task1");
IReduceFunction <int> reduceFunction = TangFactory.GetTang().NewInjector(conf2).GetInstance <IReduceFunction <int> >();
Assert.Equal(10, reduceFunction.Reduce(new int[] { 1, 2, 3, 4 }));
}
private ReduceReceiver(
[Parameter(typeof(GroupCommConfigurationOptions.OperatorName))] string operatorName,
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.Initialize))] bool initialize,
OperatorTopology <PipelineMessage <T> > topology,
IReduceFunction <T> reduceFunction,
IPipelineDataConverter <T> dataConverter)
{
OperatorName = operatorName;
GroupName = groupName;
Version = PipelineVersion;
ReduceFunction = reduceFunction;
PipelineDataConverter = dataConverter;
_initialize = initialize;
_pipelinedReduceFunc = new PipelinedReduceFunction <T>(ReduceFunction);
_topology = topology;
}
public StreamGroupedReduce(IReduceFunction <TInput> userFunction, TypeSerializer <TInput> serializer)
: base(userFunction) => _serializer = serializer;
/// <summary>
/// Applies the given window function to each window. The window function is called for each evaluation of the window for each key individually.The output of the window function is interpreted as a regular non-windowed stream.
/// Arriving data is incrementally aggregated using the given reducer.
/// </summary>
/// <typeparam name="TOutput"></typeparam>
/// <param name="reduceFunction">The reduce function that is used for incremental aggregation.</param>
/// <param name="function">The window function.</param>
/// <returns>The data stream that is the result of applying the window function to the window.</returns>
public SingleOutputStreamOperator <TOutput> Reduce <TOutput>(IReduceFunction <TElement> reduceFunction,
ProcessWindowFunction <TElement, TOutput, TKey, TWindow> function)
{
return(null);
}
public PipelinedReduceFunction(IReduceFunction <T> baseReduceFunc)
{
_baseReduceFunc = baseReduceFunc;
}
/// <summary>
/// Applies a reduce transformation on the grouped data stream grouped on by the given key position. The <see cref="IReduceFunction{T}"/> will receive input values based on the key value. Only input values with the same key will go to the same reducer.
/// </summary>
/// <param name="reducer">The <see cref="IReduceFunction{T}"/> that will be called for every element of the input values with the same key.</param>
/// <returns>The transformed DataStream.</returns>
public SingleOutputStreamOperator <TElement> Reduce(IReduceFunction <TElement> reducer)
{
return(null);
}
