public void TestScatterOperator2() { string groupName = "group1"; string operatorName = "scatter"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 5; int fanOut = 2; var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup .AddScatter <int>(operatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig()) .Build(); var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(operatorName); IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(operatorName); IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(operatorName); IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(operatorName); IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(operatorName); Assert.NotNull(sender); Assert.NotNull(receiver1); Assert.NotNull(receiver2); Assert.NotNull(receiver3); Assert.NotNull(receiver4); List <int> data = new List <int> { 1, 2, 3, 4, 5, 6, 7, 8 }; sender.Send(data); var data1 = receiver1.Receive().ToArray(); Assert.Equal(1, data1[0]); Assert.Equal(2, data1[1]); Assert.Equal(3, data1[2]); Assert.Equal(4, data1[3]); var data2 = receiver2.Receive().ToArray(); Assert.Equal(5, data2[0]); Assert.Equal(6, data2[1]); Assert.Equal(7, data2[2]); Assert.Equal(8, data2[3]); var data3 = receiver3.Receive(); Assert.Equal(1, data3.First()); Assert.Equal(2, data3.Last()); var data4 = receiver4.Receive(); Assert.Equal(3, data4.First()); Assert.Equal(4, data4.Last()); }
public byte[] Call(byte[] memento) { List <int> data = Enumerable.Range(1, 100).ToList(); _scatterSender.Send(data); int sum = _sumReducer.Reduce(); Logger.Log(Level.Info, "Received sum: {0}", sum); return(null); }
public void TestScatterOperator4() { string groupName = "group1"; string operatorName = "scatter"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 4; int fanOut = 2; IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); var commGroup = groupCommDriver.DefaultGroup .AddScatter <int>(operatorName, masterTaskId, TopologyTypes.Flat, GetDefaulDataConverterConfig()) .Build(); List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(operatorName); IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(operatorName); IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(operatorName); IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(operatorName); Assert.IsNotNull(sender); Assert.IsNotNull(receiver1); Assert.IsNotNull(receiver2); Assert.IsNotNull(receiver3); List <int> data = new List <int> { 1, 2, 3, 4, 5, 6, 7, 8 }; List <string> order = new List <string> { "task3", "task2", "task1" }; sender.Send(data, order); var data3 = receiver3.Receive().ToArray(); Assert.AreEqual(1, data3[0]); Assert.AreEqual(2, data3[1]); Assert.AreEqual(3, data3[2]); var data2 = receiver2.Receive().ToArray(); Assert.AreEqual(4, data2[0]); Assert.AreEqual(5, data2[1]); Assert.AreEqual(6, data2[2]); var data1 = receiver1.Receive().ToArray(); Assert.AreEqual(7, data1[0]); Assert.AreEqual(8, data1[1]); }
public void TestScatterOperatorWithDefaultCodec() { string groupName = "group1"; string operatorName = "scatter"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 5; int fanOut = 2; IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); var commGroup = groupCommDriver.DefaultGroup .AddScatter(operatorName, masterTaskId) .Build(); List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(operatorName); IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(operatorName); IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(operatorName); IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(operatorName); IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(operatorName); Assert.NotNull(sender); Assert.NotNull(receiver1); Assert.NotNull(receiver2); Assert.NotNull(receiver3); Assert.NotNull(receiver4); List <int> data = new List <int> { 1, 2, 3, 4 }; sender.Send(data); Assert.Equal(1, receiver1.Receive().Single()); Assert.Equal(2, receiver2.Receive().Single()); Assert.Equal(3, receiver3.Receive().Single()); Assert.Equal(4, receiver4.Receive().Single()); }
public void TestScatterReduceOperators() { string groupName = "group1"; string scatterOperatorName = "scatter"; string reduceOperatorName = "reduce"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 5; int fanOut = 2; IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup .AddScatter <int>( scatterOperatorName, masterTaskId, TopologyTypes.Flat, GetDefaultDataConverterConfig()) .AddReduce <int>( reduceOperatorName, masterTaskId, TopologyTypes.Flat, GetDefaultReduceFuncConfig(), GetDefaultDataConverterConfig()) .Build(); List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(scatterOperatorName); IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName); IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName); IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName); IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender3 = commGroups[3].GetReduceSender <int>(reduceOperatorName); IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender4 = commGroups[4].GetReduceSender <int>(reduceOperatorName); Assert.NotNull(sender); Assert.NotNull(receiver1); Assert.NotNull(receiver2); Assert.NotNull(receiver3); Assert.NotNull(receiver4); List <int> data = Enumerable.Range(1, 100).ToList(); List <string> order = new List <string> { "task4", "task3", "task2", "task1" }; sender.Send(data, order); ScatterReceiveReduce(receiver4, sumSender4); ScatterReceiveReduce(receiver3, sumSender3); ScatterReceiveReduce(receiver2, sumSender2); ScatterReceiveReduce(receiver1, sumSender1); int sum = sumReducer.Reduce(); Assert.Equal(sum, data.Sum()); }
public void TestScatterReduceOperatorsWithCancelation() { string groupName = "group1"; string scatterOperatorName = "scatter"; string reduceOperatorName = "reduce"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 5; int fanOut = 2; var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup .AddScatter <int>( scatterOperatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig()) .AddReduce <int>( reduceOperatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig(), GetDefaultReduceFuncConfig()) .Build(); var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(scatterOperatorName); IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName); IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName); IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName); IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender3 = commGroups[3].GetReduceSender <int>(reduceOperatorName); IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender4 = commGroups[4].GetReduceSender <int>(reduceOperatorName); Assert.NotNull(sender); Assert.NotNull(receiver1); Assert.NotNull(receiver2); Assert.NotNull(receiver3); Assert.NotNull(receiver4); List <int> data = Enumerable.Range(1, 100).ToList(); sender.Send(data); List <int> data1 = receiver1.Receive(); List <int> data2 = receiver2.Receive(); List <int> data3 = receiver3.Receive(); List <int> data4 = receiver4.Receive(); int sum3 = data3.Sum(); sumSender3.Send(sum3); int sum4 = data4.Sum(); sumSender4.Send(sum4); int sum2 = data2.Sum(); sumSender2.Send(sum2); int sum1 = data1.Sum(); var token = new CancellationTokenSource(); token.Cancel(); Action send = () => sumSender1.Send(sum1, token); Assert.Throws <OperationCanceledException>(send); var taskThread = new Thread(() => { Action reduce = () => sumReducer.Reduce(token); Assert.Throws <OperationCanceledException>(reduce); }); taskThread.Start(); token.Cancel(); }
public void TestScatterReduceOperators() { string groupName = "group1"; string scatterOperatorName = "scatter"; string reduceOperatorName = "reduce"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 5; int fanOut = 2; var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup .AddScatter <int>( scatterOperatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig()) .AddReduce <int>( reduceOperatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig(), GetDefaultReduceFuncConfig()) .Build(); var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(scatterOperatorName); IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName); IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName); IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName); IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender3 = commGroups[3].GetReduceSender <int>(reduceOperatorName); IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(scatterOperatorName); IReduceSender <int> sumSender4 = commGroups[4].GetReduceSender <int>(reduceOperatorName); Assert.IsNotNull(sender); Assert.IsNotNull(receiver1); Assert.IsNotNull(receiver2); Assert.IsNotNull(receiver3); Assert.IsNotNull(receiver4); List <int> data = Enumerable.Range(1, 100).ToList(); sender.Send(data); List <int> data1 = receiver1.Receive(); List <int> data2 = receiver2.Receive(); List <int> data3 = receiver3.Receive(); List <int> data4 = receiver4.Receive(); int sum3 = data3.Sum(); sumSender3.Send(sum3); int sum4 = data4.Sum(); sumSender4.Send(sum4); int sum2 = data2.Sum(); sumSender2.Send(sum2); int sum1 = data1.Sum(); sumSender1.Send(sum1); int sum = sumReducer.Reduce(); Assert.AreEqual(sum, 6325); }