private static void ScatterReceiveReduce(IScatterReceiver <int> receiver, IReduceSender <int> sumSender)
{
List <int> data1 = receiver.Receive();
int sum1 = data1.Sum();
sumSender.Send(sum1);
}
public void TestScatterOperator2()
{
string groupName = "group1";
string operatorName = "scatter";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 5;
int fanOut = 2;
var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup
.AddScatter <int>(operatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig())
.Build();
var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(operatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(operatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
Assert.NotNull(receiver3);
Assert.NotNull(receiver4);
List <int> data = new List <int> {
1, 2, 3, 4, 5, 6, 7, 8
};
sender.Send(data);
var data1 = receiver1.Receive().ToArray();
Assert.Equal(1, data1[0]);
Assert.Equal(2, data1[1]);
Assert.Equal(3, data1[2]);
Assert.Equal(4, data1[3]);
var data2 = receiver2.Receive().ToArray();
Assert.Equal(5, data2[0]);
Assert.Equal(6, data2[1]);
Assert.Equal(7, data2[2]);
Assert.Equal(8, data2[3]);
var data3 = receiver3.Receive();
Assert.Equal(1, data3.First());
Assert.Equal(2, data3.Last());
var data4 = receiver4.Receive();
Assert.Equal(3, data4.First());
Assert.Equal(4, data4.Last());
}
public SlaveTask(IGroupCommClient groupCommClient)
{
_logger.Log(Level.Info, "Hello from slave task");
_groupCommClient = groupCommClient;
_commGroup = _groupCommClient.GetCommunicationGroup(GroupTestConstants.GroupName);
_scatterReceiver = _commGroup.GetScatterReceiver <int>(GroupTestConstants.ScatterOperatorName);
_sumSender = _commGroup.GetReduceSender <int>(GroupTestConstants.ReduceOperatorName);
}
public SlaveTask(IGroupCommClient groupCommClient)
{
Logger.Log(Level.Info, "Hello from slave task");
_groupCommClient = groupCommClient;
_commGroup = _groupCommClient.GetCommunicationGroup(GroupTestConstants.GroupName);
_scatterReceiver = _commGroup.GetScatterReceiver<int>(GroupTestConstants.ScatterOperatorName);
_sumSender = _commGroup.GetReduceSender<int>(GroupTestConstants.ReduceOperatorName);
}
public void TestScatterOperator4()
{
string groupName = "group1";
string operatorName = "scatter";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 4;
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddScatter <int>(operatorName, masterTaskId, TopologyTypes.Flat, GetDefaulDataConverterConfig())
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(operatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(operatorName);
Assert.IsNotNull(sender);
Assert.IsNotNull(receiver1);
Assert.IsNotNull(receiver2);
Assert.IsNotNull(receiver3);
List <int> data = new List <int> {
1, 2, 3, 4, 5, 6, 7, 8
};
List <string> order = new List <string> {
"task3", "task2", "task1"
};
sender.Send(data, order);
var data3 = receiver3.Receive().ToArray();
Assert.AreEqual(1, data3[0]);
Assert.AreEqual(2, data3[1]);
Assert.AreEqual(3, data3[2]);
var data2 = receiver2.Receive().ToArray();
Assert.AreEqual(4, data2[0]);
Assert.AreEqual(5, data2[1]);
Assert.AreEqual(6, data2[2]);
var data1 = receiver1.Receive().ToArray();
Assert.AreEqual(7, data1[0]);
Assert.AreEqual(8, data1[1]);
}
public void TestScatterOperatorWithDefaultCodec()
{
string groupName = "group1";
string operatorName = "scatter";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 5;
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddScatter(operatorName, masterTaskId)
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(operatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(operatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
Assert.NotNull(receiver3);
Assert.NotNull(receiver4);
List <int> data = new List <int> {
1, 2, 3, 4
};
sender.Send(data);
Assert.Equal(1, receiver1.Receive().Single());
Assert.Equal(2, receiver2.Receive().Single());
Assert.Equal(3, receiver3.Receive().Single());
Assert.Equal(4, receiver4.Receive().Single());
}
public void TestScatterReduceOperators()
{
string groupName = "group1";
string scatterOperatorName = "scatter";
string reduceOperatorName = "reduce";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 5;
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup
.AddScatter <int>(
scatterOperatorName,
masterTaskId,
TopologyTypes.Flat,
GetDefaultDataConverterConfig())
.AddReduce <int>(
reduceOperatorName,
masterTaskId,
TopologyTypes.Flat,
GetDefaultReduceFuncConfig(),
GetDefaultDataConverterConfig())
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(scatterOperatorName);
IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender3 = commGroups[3].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender4 = commGroups[4].GetReduceSender <int>(reduceOperatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
Assert.NotNull(receiver3);
Assert.NotNull(receiver4);
List <int> data = Enumerable.Range(1, 100).ToList();
List <string> order = new List <string> {
"task4", "task3", "task2", "task1"
};
sender.Send(data, order);
ScatterReceiveReduce(receiver4, sumSender4);
ScatterReceiveReduce(receiver3, sumSender3);
ScatterReceiveReduce(receiver2, sumSender2);
ScatterReceiveReduce(receiver1, sumSender1);
int sum = sumReducer.Reduce();
Assert.Equal(sum, data.Sum());
}
public void TestScatterReduceOperatorsWithCancelation()
{
string groupName = "group1";
string scatterOperatorName = "scatter";
string reduceOperatorName = "reduce";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 5;
int fanOut = 2;
var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup
.AddScatter <int>(
scatterOperatorName,
masterTaskId,
TopologyTypes.Tree,
GetDefaultDataConverterConfig())
.AddReduce <int>(
reduceOperatorName,
masterTaskId,
TopologyTypes.Tree,
GetDefaultDataConverterConfig(),
GetDefaultReduceFuncConfig())
.Build();
var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(scatterOperatorName);
IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender3 = commGroups[3].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender4 = commGroups[4].GetReduceSender <int>(reduceOperatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
Assert.NotNull(receiver3);
Assert.NotNull(receiver4);
List <int> data = Enumerable.Range(1, 100).ToList();
sender.Send(data);
List <int> data1 = receiver1.Receive();
List <int> data2 = receiver2.Receive();
List <int> data3 = receiver3.Receive();
List <int> data4 = receiver4.Receive();
int sum3 = data3.Sum();
sumSender3.Send(sum3);
int sum4 = data4.Sum();
sumSender4.Send(sum4);
int sum2 = data2.Sum();
sumSender2.Send(sum2);
int sum1 = data1.Sum();
var token = new CancellationTokenSource();
token.Cancel();
Action send = () => sumSender1.Send(sum1, token);
Assert.Throws <OperationCanceledException>(send);
var taskThread = new Thread(() =>
{
Action reduce = () => sumReducer.Reduce(token);
Assert.Throws <OperationCanceledException>(reduce);
});
taskThread.Start();
token.Cancel();
}
public void TestScatterReduceOperators()
{
string groupName = "group1";
string scatterOperatorName = "scatter";
string reduceOperatorName = "reduce";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 5;
int fanOut = 2;
var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup
.AddScatter <int>(
scatterOperatorName,
masterTaskId,
TopologyTypes.Tree,
GetDefaultDataConverterConfig())
.AddReduce <int>(
reduceOperatorName,
masterTaskId,
TopologyTypes.Tree,
GetDefaultDataConverterConfig(),
GetDefaultReduceFuncConfig())
.Build();
var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(scatterOperatorName);
IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender3 = commGroups[3].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender4 = commGroups[4].GetReduceSender <int>(reduceOperatorName);
Assert.IsNotNull(sender);
Assert.IsNotNull(receiver1);
Assert.IsNotNull(receiver2);
Assert.IsNotNull(receiver3);
Assert.IsNotNull(receiver4);
List <int> data = Enumerable.Range(1, 100).ToList();
sender.Send(data);
List <int> data1 = receiver1.Receive();
List <int> data2 = receiver2.Receive();
List <int> data3 = receiver3.Receive();
List <int> data4 = receiver4.Receive();
int sum3 = data3.Sum();
sumSender3.Send(sum3);
int sum4 = data4.Sum();
sumSender4.Send(sum4);
int sum2 = data2.Sum();
sumSender2.Send(sum2);
int sum1 = data1.Sum();
sumSender1.Send(sum1);
int sum = sumReducer.Reduce();
Assert.AreEqual(sum, 6325);
}
