private ReduceReceiver(
[Parameter(typeof(GroupCommConfigurationOptions.OperatorName))] string operatorName,
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.Initialize))] bool initialize,
OperatorTopology <PipelineMessage <T> > topology,
ICommunicationGroupNetworkObserver networkHandler,
IReduceFunction <T> reduceFunction,
IPipelineDataConverter <T> dataConverter)
{
OperatorName = operatorName;
GroupName = groupName;
Version = PipelineVersion;
ReduceFunction = reduceFunction;
PipelineDataConverter = dataConverter;
_pipelinedReduceFunc = new PipelinedReduceFunction <T>(ReduceFunction);
_topology = topology;
var msgHandler = Observer.Create <GeneralGroupCommunicationMessage>(message => topology.OnNext(message));
networkHandler.Register(operatorName, msgHandler);
if (initialize)
{
topology.Initialize();
}
}
/// <summary>
/// Verify that the FullMessage method properly handles null and zero-length arrays
/// </summary>
/// <typeparam name="T">The type of array the IPipelineDataConverter converts</typeparam>
/// <typeparam name="DataConverter">The IPipelineDataConverter implementation</typeparam>
private static void AssertNullAndZeroLengthArraysHandledByFullMessage <T, DataConverter>()
where T : new()
where DataConverter : class, IPipelineDataConverter <T[]>
{
// Test that the valid configuration can be injected
IConfiguration config = GetPipelineDataConverterConfig(1);
IPipelineDataConverter <T[]> dataConverter = TangFactory.GetTang().NewInjector(config).GetInstance <ArrayPipelineDataConverter <T> >();
// Test that a null message returns a null object
List <PipelineMessage <T[]> > nullMessage = new List <PipelineMessage <T[]> >
{
new PipelineMessage <T[]>(null, true)
};
Assert.Null(dataConverter.FullMessage(nullMessage));
// Test that many null messages returns a null object
List <PipelineMessage <T[]> > manyNullMessages = new List <PipelineMessage <T[]> >
{
new PipelineMessage <T[]>(null, false),
new PipelineMessage <T[]>(null, true)
};
Assert.Null(dataConverter.FullMessage(manyNullMessages));
// Test that null messages mixed with non-null returns the non-null object
List <PipelineMessage <T[]> > someNullMessages = new List <PipelineMessage <T[]> >
{
new PipelineMessage <T[]>(null, false),
new PipelineMessage <T[]>(new T[2], false),
new PipelineMessage <T[]>(null, true)
};
Assert.Equal(2, dataConverter.FullMessage(someNullMessages).Length);
// Test that a zero-length message returns a zero-length object
List <PipelineMessage <T[]> > zeroLengthMessage = new List <PipelineMessage <T[]> >
{
new PipelineMessage <T[]>(new T[0], true)
};
Assert.Equal(0, dataConverter.FullMessage(zeroLengthMessage).Length);
// Test that many zero-length message return a zero-length object
List <PipelineMessage <T[]> > manyZeroLengthMessages = new List <PipelineMessage <T[]> >
{
new PipelineMessage <T[]>(new T[0], false),
new PipelineMessage <T[]>(new T[0], true)
};
Assert.Equal(0, dataConverter.FullMessage(manyZeroLengthMessages).Length);
}
private BroadcastSender(
[Parameter(typeof(GroupCommConfigurationOptions.OperatorName))] string operatorName,
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.Initialize))] bool initialize,
OperatorTopology <PipelineMessage <T> > topology,
IPipelineDataConverter <T> dataConverter)
{
_topology = topology;
OperatorName = operatorName;
GroupName = groupName;
Version = PipelineVersion;
PipelineDataConverter = dataConverter;
_initialize = initialize;
}
public void TestMapInputPipelining()
{
int chunkSize = 2;
var config = TangFactory.GetTang().NewConfigurationBuilder(
PipelineDataConverterConfiguration <int[]> .Conf
.Set(PipelineDataConverterConfiguration <int[]> .DataConverter,
GenericType <PipelineIntDataConverter> .Class)
.Build()).BindNamedParameter <ChunkSize, int>(
GenericType <ChunkSize> .Class,
chunkSize.ToString(CultureInfo.InvariantCulture)).Build();
IPipelineDataConverter <MapInputWithControlMessage <int[]> > dataConverter =
TangFactory.GetTang()
.NewInjector(config)
.GetInstance <MapInputwithControlMessagePipelineDataConverter <int[]> >();
int[] baseMessage = { 1, 2, 3 };
var chunks1 = dataConverter.PipelineMessage(new MapInputWithControlMessage <int[]>(baseMessage,
MapControlMessage.AnotherRound));
var chunks2 = dataConverter.PipelineMessage(new MapInputWithControlMessage <int[]>(MapControlMessage.Stop));
Assert.Equal(chunks1.Count, 2);
Assert.True(chunks1[0].Data.Message.Length == 2);
Assert.True(chunks1[1].Data.Message.Length == 1);
Assert.Equal(chunks1[0].Data.Message[0], baseMessage[0]);
Assert.Equal(chunks1[0].Data.Message[1], baseMessage[1]);
Assert.Equal(chunks1[1].Data.Message[0], baseMessage[2]);
Assert.Equal(chunks1[0].Data.ControlMessage, MapControlMessage.AnotherRound);
Assert.Equal(chunks1[1].Data.ControlMessage, MapControlMessage.AnotherRound);
Assert.Equal(chunks1[0].IsLast, false);
Assert.Equal(chunks1[1].IsLast, true);
Assert.Equal(chunks2.Count, 1);
Assert.Null(chunks2[0].Data.Message);
Assert.Equal(chunks2[0].Data.ControlMessage, MapControlMessage.Stop);
Assert.Equal(chunks2[0].IsLast, true);
var fullMessage1 = dataConverter.FullMessage(chunks1);
var fullMessage2 = dataConverter.FullMessage(chunks2);
Assert.Equal(fullMessage1.Message[0], baseMessage[0]);
Assert.Equal(fullMessage1.Message[1], baseMessage[1]);
Assert.Equal(fullMessage1.Message[2], baseMessage[2]);
Assert.Equal(fullMessage1.ControlMessage, chunks1[0].Data.ControlMessage);
Assert.Null(fullMessage2.Message);
Assert.Equal(fullMessage2.ControlMessage, chunks2[0].Data.ControlMessage);
}
/// <summary>
/// Master test function for testing types of arrays in the ArrayPipelineDataConverter
/// </summary>
/// <typeparam name="T">The type of array to test</typeparam>
/// <param name="originalArray">An array to use in the test</param>
private static void TestArrayPipelineDataConverter <T>(T[] originalArray) where T : new()
{
// Verify that the constructor has the proper restrictions
AssertPositivePipelinePackageElementsRequired <T[], ArrayPipelineDataConverter <T> >();
// Verify that the FullMessage method properly handles null and zero-length arrays
AssertNullAndZeroLengthArraysHandledByFullMessage <T, ArrayPipelineDataConverter <T> >();
// Test the valid case where we break up the array into smaller pieces
// First determine how many messages to create from originalArray
int pipelineMessageSize;
int nMessages;
if (originalArray == null)
{
nMessages = 0;
pipelineMessageSize = 1;
}
else if (originalArray.Length == 0 || originalArray.Length == 1)
{
nMessages = 1;
pipelineMessageSize = 1; // Necessary to instantiate the ArrayPipelineDataConverterConfig
}
else
{
nMessages = 2;
pipelineMessageSize = (int)Math.Ceiling(originalArray.Length / (double)nMessages);
}
// Test that the valid configuration can be injected
IConfiguration config = GetPipelineDataConverterConfig(pipelineMessageSize);
IPipelineDataConverter <T[]> dataConverter = TangFactory.GetTang().NewInjector(config).GetInstance <ArrayPipelineDataConverter <T> >();
var pipelineData = dataConverter.PipelineMessage(originalArray);
// Validate that the pipeline constructed the correct number of messages
Assert.Equal <int>(pipelineData.Count, nMessages);
T[] deserializedArray = dataConverter.FullMessage(pipelineData);
// Validate that the array is unaffected by the serialization / deserialization
AssertArrayEquality(originalArray, deserializedArray);
// Verify that the "IsLast" property is set correctly
AssertIsLastFlag(pipelineData);
}
private ReduceReceiver(
[Parameter(typeof(GroupCommConfigurationOptions.OperatorName))] string operatorName,
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.Initialize))] bool initialize,
OperatorTopology <PipelineMessage <T> > topology,
IReduceFunction <T> reduceFunction,
IPipelineDataConverter <T> dataConverter)
{
OperatorName = operatorName;
GroupName = groupName;
Version = PipelineVersion;
ReduceFunction = reduceFunction;
PipelineDataConverter = dataConverter;
_initialize = initialize;
_pipelinedReduceFunc = new PipelinedReduceFunction <T>(ReduceFunction);
_topology = topology;
}
private BroadcastSender(
[Parameter(typeof(GroupCommConfigurationOptions.OperatorName))] string operatorName,
[Parameter(typeof(GroupCommConfigurationOptions.CommunicationGroupName))] string groupName,
[Parameter(typeof(GroupCommConfigurationOptions.Initialize))] bool initialize,
OperatorTopology <PipelineMessage <T> > topology,
ICommunicationGroupNetworkObserver networkHandler,
IPipelineDataConverter <T> dataConverter)
{
_topology = topology;
OperatorName = operatorName;
GroupName = groupName;
Version = PipelineVersion;
PipelineDataConverter = dataConverter;
_initialize = initialize;
var msgHandler = Observer.Create <GeneralGroupCommunicationMessage>(message => topology.OnNext(message));
networkHandler.Register(operatorName, msgHandler);
}
internal MapInputwithControlMessagePipelineDataConverter(
IPipelineDataConverter <TMapInput> basePipelineDataConverter)
{
_basePipelineDataConverter = basePipelineDataConverter;
}
