/// <summary> /// Performs IMRU iterations on update side /// </summary> /// <returns></returns> protected override byte[] TaskBody(byte[] memento) { UpdateResult <TMapInput, TResult> updateResult = null; try { updateResult = _updateTask.Initialize(); } catch (Exception e) { HandleTaskAppException(e); } while (!_cancellationSource.IsCancellationRequested && updateResult.HasMapInput) { using ( var message = new MapInputWithControlMessage <TMapInput>(updateResult.MapInput, MapControlMessage.AnotherRound)) { _dataAndControlMessageSender.Send(message); } if (_invokeGc) { Logger.Log(Level.Verbose, "Calling Garbage Collector"); GC.Collect(); GC.WaitForPendingFinalizers(); } var input = _dataReceiver.Reduce(_cancellationSource); try { updateResult = _updateTask.Update(input); if (updateResult.HasResult) { _resultHandler.HandleResult(updateResult.Result); _done = true; } } catch (Exception e) { HandleTaskAppException(e); } } if (!_cancellationSource.IsCancellationRequested) { MapInputWithControlMessage <TMapInput> stopMessage = new MapInputWithControlMessage <TMapInput>(MapControlMessage.Stop); _dataAndControlMessageSender.Send(stopMessage); } if (_done) { return(ByteUtilities.StringToByteArrays(TaskManager.UpdateTaskCompleted)); } return(null); }
public void TestBroadcastOperatorWithDefaultCodec() { INameServer nameServer = NameServerTests.BuildNameServer(); string groupName = "group1"; string operatorName = "broadcast"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 10; int value = 1337; int fanOut = 3; IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); var commGroup = groupCommDriver.DefaultGroup .AddBroadcast(operatorName, masterTaskId) .Build(); List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName); IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName); Assert.NotNull(sender); Assert.NotNull(receiver1); Assert.NotNull(receiver2); sender.Send(value); Assert.Equal(value, receiver1.Receive()); Assert.Equal(value, receiver2.Receive()); }
public byte[] Call(byte[] memento) { int[] intArr = new int[_arraySize]; for (int i = 1; i <= _numIters; i++) { for (int j = 0; j < _arraySize; j++) { intArr[j] = i; } _broadcastSender.Send(intArr); int[] sum = _sumReducer.Reduce(); Logger.Log(Level.Info, "Received sum: {0} on iteration: {1} with array length: {2}", sum[0], i, sum.Length); int expected = TriangleNumber(i) * _numReduceSenders; for (int j = 0; j < intArr.Length; j++) { if (sum[j] != TriangleNumber(i) * _numReduceSenders) { throw new Exception("Expected " + expected + " but got " + sum); } } } return(null); }
/// <summary> /// Performs IMRU iterations on update side /// </summary> /// <param name="memento"></param> /// <returns></returns> public byte[] Call(byte[] memento) { var updateResult = _updateTask.Initialize(); int iterNo = 0; while (updateResult.HasMapInput && !_taskCloseCoordinator.ShouldCloseTask()) { iterNo++; using ( var message = new MapInputWithControlMessage <TMapInput>(updateResult.MapInput, MapControlMessage.AnotherRound)) { _dataAndControlMessageSender.Send(message); } var input = _dataReceiver.Reduce(); if (_invokeGC) { Logger.Log(Level.Verbose, "Calling Garbage Collector"); GC.Collect(); GC.WaitForPendingFinalizers(); } updateResult = _updateTask.Update(input); if (updateResult.HasResult) { _resultHandler.HandleResult(updateResult.Result); } } if (!_taskCloseCoordinator.ShouldCloseTask()) { MapInputWithControlMessage <TMapInput> stopMessage = new MapInputWithControlMessage <TMapInput>(MapControlMessage.Stop); _dataAndControlMessageSender.Send(stopMessage); } _resultHandler.Dispose(); _taskCloseCoordinator.SignalTaskStopped(); return(null); }
public void TestBroadcastReduceOperators() { string groupName = "group1"; string broadcastOperatorName = "broadcast"; string reduceOperatorName = "reduce"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 3; int fanOut = 2; var groupCommunicationDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); ICommunicationGroupDriver commGroup = groupCommunicationDriver.DefaultGroup .AddBroadcast <int>( broadcastOperatorName, masterTaskId, TopologyTypes.Flat, GetDefaultDataConverterConfig()) .AddReduce <int>( reduceOperatorName, masterTaskId, TopologyTypes.Flat, GetDefaultDataConverterConfig(), GetDefaultReduceFuncConfig()) .Build(); var commGroups = CommGroupClients(groupName, numTasks, groupCommunicationDriver, commGroup, GetDefaultCodecConfig()); // for master task IBroadcastSender <int> broadcastSender = commGroups[0].GetBroadcastSender <int>(broadcastOperatorName); IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver1 = commGroups[1].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver2 = commGroups[2].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName); for (int j = 1; j <= 10; j++) { broadcastSender.Send(j); int n1 = broadcastReceiver1.Receive(); int n2 = broadcastReceiver2.Receive(); Assert.Equal(j, n1); Assert.Equal(j, n2); int triangleNum1 = TriangleNumber(n1); triangleNumberSender1.Send(triangleNum1); int triangleNum2 = TriangleNumber(n2); triangleNumberSender2.Send(triangleNum2); int sum = sumReducer.Reduce(); int expected = TriangleNumber(j) * (numTasks - 1); Assert.Equal(sum, expected); } }
public void TestBroadcastOperator2() { string groupName = "group1"; string operatorName = "broadcast"; string driverId = "driverId"; string masterTaskId = "task0"; int numTasks = 3; int value1 = 1337; int value2 = 42; int value3 = 99; int fanOut = 2; IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); var commGroup = groupCommDriver.DefaultGroup .AddBroadcast(operatorName, masterTaskId) .Build(); List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName); IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName); Assert.IsNotNull(sender); Assert.IsNotNull(receiver1); Assert.IsNotNull(receiver2); sender.Send(value1); Assert.AreEqual(value1, receiver1.Receive()); Assert.AreEqual(value1, receiver2.Receive()); sender.Send(value2); Assert.AreEqual(value2, receiver1.Receive()); Assert.AreEqual(value2, receiver2.Receive()); sender.Send(value3); Assert.AreEqual(value3, receiver1.Receive()); Assert.AreEqual(value3, receiver2.Receive()); }
public byte[] Call(byte[] memento) { _groupCommClient.Initialize(); int[] intArr = new int[_arraySize]; for (int j = 0; j < _arraySize; j++) { intArr[j] = j; } Stopwatch broadcastTime = new Stopwatch(); Stopwatch reduceTime = new Stopwatch(); for (int i = 1; i <= _numIters; i++) { intArr[0] = i; if (i == 2) { broadcastTime.Reset(); reduceTime.Reset(); } broadcastTime.Start(); _broadcastSender.Send(intArr); broadcastTime.Stop(); reduceTime.Start(); int[] sum = _sumReducer.Reduce(); reduceTime.Stop(); Logger.Log(Level.Info, "Received sum: {0} on iteration: {1} with array length: {2}", sum[0], i, sum.Length); int expected = TriangleNumber(i) * _numReduceSenders; if (sum[0] != TriangleNumber(i) * _numReduceSenders) { throw new Exception("Expected " + expected + " but got " + sum[0]); } if (i >= 2) { var msg = string.Format("Average time (milliseconds) taken for broadcast: {0} and reduce: {1}", broadcastTime.ElapsedMilliseconds / ((double)(i - 1)), reduceTime.ElapsedMilliseconds / ((double)(i - 1))); Logger.Log(Level.Info, msg); } } return(null); }
public byte[] Call(byte[] memento) { _groupCommClient.Initialize(); Stopwatch broadcastTime = new Stopwatch(); Stopwatch reduceTime = new Stopwatch(); for (int i = 1; i <= _numIters; i++) { if (i == 2) { broadcastTime.Reset(); reduceTime.Reset(); } broadcastTime.Start(); // Each slave task calculates the nth triangle number _broadcastSender.Send(i); broadcastTime.Stop(); reduceTime.Start(); // Sum up all of the calculated triangle numbers int sum = _sumReducer.Reduce(); reduceTime.Stop(); Logger.Log(Level.Info, "Received sum: {0} on iteration: {1}", sum, i); int expected = TriangleNumber(i) * _numReduceSenders; if (sum != TriangleNumber(i) * _numReduceSenders) { throw new Exception("Expected " + expected + " but got " + sum); } if (i >= 2) { var msg = string.Format("Average time (milliseconds) taken for broadcast: {0} and reduce: {1}", broadcastTime.ElapsedMilliseconds / ((double)(i - 1)), reduceTime.ElapsedMilliseconds / ((double)(i - 1))); Logger.Log(Level.Info, msg); } } return(null); }
public byte[] Call(byte[] memento) { for (int i = 1; i <= _numIters; i++) { // Each slave task calculates the nth triangle number _broadcastSender.Send(i); // Sum up all of the calculated triangle numbers int sum = _sumReducer.Reduce(); _logger.Log(Level.Info, "Received sum: {0} on iteration: {1}", sum, i); int expected = TriangleNumber(i) * _numReduceSenders; if (sum != TriangleNumber(i) * _numReduceSenders) { throw new Exception("Expected " + expected + " but got " + sum); } } return(null); }
public void TestBroadcastReduceOperators() { string groupName = "group1"; string broadcastOperatorName = "broadcast"; string reduceOperatorName = "reduce"; string masterTaskId = "task0"; string driverId = "Driver Id"; int numTasks = 10; int fanOut = 3; var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup .AddBroadcast <int>( broadcastOperatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig()) .AddReduce <int>( reduceOperatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig(), GetDefaultReduceFuncConfig()) .Build(); var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); // for master task IBroadcastSender <int> broadcastSender = commGroups[0].GetBroadcastSender <int>(broadcastOperatorName); IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver1 = commGroups[1].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver2 = commGroups[2].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver3 = commGroups[3].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender3 = commGroups[3].GetReduceSender <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver4 = commGroups[4].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender4 = commGroups[4].GetReduceSender <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver5 = commGroups[5].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender5 = commGroups[5].GetReduceSender <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver6 = commGroups[6].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender6 = commGroups[6].GetReduceSender <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver7 = commGroups[7].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender7 = commGroups[7].GetReduceSender <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver8 = commGroups[8].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender8 = commGroups[8].GetReduceSender <int>(reduceOperatorName); IBroadcastReceiver <int> broadcastReceiver9 = commGroups[9].GetBroadcastReceiver <int>(broadcastOperatorName); IReduceSender <int> triangleNumberSender9 = commGroups[9].GetReduceSender <int>(reduceOperatorName); for (int i = 1; i <= 10; i++) { broadcastSender.Send(i); int n1 = broadcastReceiver1.Receive(); int n2 = broadcastReceiver2.Receive(); int n3 = broadcastReceiver3.Receive(); int n4 = broadcastReceiver4.Receive(); int n5 = broadcastReceiver5.Receive(); int n6 = broadcastReceiver6.Receive(); int n7 = broadcastReceiver7.Receive(); int n8 = broadcastReceiver8.Receive(); int n9 = broadcastReceiver9.Receive(); Assert.Equal(i, n1); Assert.Equal(i, n2); Assert.Equal(i, n3); Assert.Equal(i, n4); Assert.Equal(i, n5); Assert.Equal(i, n6); Assert.Equal(i, n7); Assert.Equal(i, n8); Assert.Equal(i, n9); int triangleNum9 = GroupCommunicationTests.TriangleNumber(n9); triangleNumberSender9.Send(triangleNum9); int triangleNum8 = GroupCommunicationTests.TriangleNumber(n8); triangleNumberSender8.Send(triangleNum8); int triangleNum7 = GroupCommunicationTests.TriangleNumber(n7); triangleNumberSender7.Send(triangleNum7); int triangleNum6 = GroupCommunicationTests.TriangleNumber(n6); triangleNumberSender6.Send(triangleNum6); int triangleNum5 = GroupCommunicationTests.TriangleNumber(n5); triangleNumberSender5.Send(triangleNum5); int triangleNum4 = GroupCommunicationTests.TriangleNumber(n4); triangleNumberSender4.Send(triangleNum4); int triangleNum3 = GroupCommunicationTests.TriangleNumber(n3); triangleNumberSender3.Send(triangleNum3); int triangleNum2 = GroupCommunicationTests.TriangleNumber(n2); triangleNumberSender2.Send(triangleNum2); int triangleNum1 = GroupCommunicationTests.TriangleNumber(n1); triangleNumberSender1.Send(triangleNum1); int sum = sumReducer.Reduce(); int expected = GroupCommunicationTests.TriangleNumber(i) * (numTasks - 1); Assert.Equal(sum, expected); } }
public void TestBroadcastOperator() { string groupName = "group1"; string operatorName = "broadcast"; string driverId = "driverId"; string masterTaskId = "task0"; int numTasks = 10; int value1 = 1337; int value2 = 42; int value3 = 99; int fanOut = 3; var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks); ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup .AddBroadcast <int>(operatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig(), GetDefaultReduceFuncConfig()) .Build(); var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig()); IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName); IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver3 = commGroups[3].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver4 = commGroups[4].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver5 = commGroups[5].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver6 = commGroups[6].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver7 = commGroups[7].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver8 = commGroups[8].GetBroadcastReceiver <int>(operatorName); IBroadcastReceiver <int> receiver9 = commGroups[9].GetBroadcastReceiver <int>(operatorName); Assert.NotNull(sender); Assert.NotNull(receiver1); Assert.NotNull(receiver2); Assert.NotNull(receiver3); Assert.NotNull(receiver4); Assert.NotNull(receiver5); Assert.NotNull(receiver6); Assert.NotNull(receiver7); Assert.NotNull(receiver8); Assert.NotNull(receiver9); sender.Send(value1); Assert.Equal(value1, receiver1.Receive()); Assert.Equal(value1, receiver2.Receive()); Assert.Equal(value1, receiver3.Receive()); Assert.Equal(value1, receiver4.Receive()); Assert.Equal(value1, receiver5.Receive()); Assert.Equal(value1, receiver6.Receive()); Assert.Equal(value1, receiver7.Receive()); Assert.Equal(value1, receiver8.Receive()); Assert.Equal(value1, receiver9.Receive()); sender.Send(value2); Assert.Equal(value2, receiver1.Receive()); Assert.Equal(value2, receiver2.Receive()); Assert.Equal(value2, receiver3.Receive()); Assert.Equal(value2, receiver4.Receive()); Assert.Equal(value2, receiver5.Receive()); Assert.Equal(value2, receiver6.Receive()); Assert.Equal(value2, receiver7.Receive()); Assert.Equal(value2, receiver8.Receive()); Assert.Equal(value2, receiver9.Receive()); sender.Send(value3); Assert.Equal(value3, receiver1.Receive()); Assert.Equal(value3, receiver2.Receive()); Assert.Equal(value3, receiver3.Receive()); Assert.Equal(value3, receiver4.Receive()); Assert.Equal(value3, receiver5.Receive()); Assert.Equal(value3, receiver6.Receive()); Assert.Equal(value3, receiver7.Receive()); Assert.Equal(value3, receiver8.Receive()); Assert.Equal(value3, receiver9.Receive()); }
/// <summary> /// Performs IMRU iterations on update side /// </summary> /// <param name="memento"></param> /// <returns></returns> public byte[] Call(byte[] memento) { var updateResult = _updateTask.Initialize(); int iterNo = 0; try { while (updateResult.HasMapInput && !_cancellationSource.IsCancellationRequested) { iterNo++; using ( var message = new MapInputWithControlMessage <TMapInput>(updateResult.MapInput, MapControlMessage.AnotherRound)) { _dataAndControlMessageSender.Send(message); } var input = _dataReceiver.Reduce(_cancellationSource); if (_invokeGC) { Logger.Log(Level.Verbose, "Calling Garbage Collector"); GC.Collect(); GC.WaitForPendingFinalizers(); } updateResult = _updateTask.Update(input); if (updateResult.HasResult) { _resultHandler.HandleResult(updateResult.Result); } } if (!_cancellationSource.IsCancellationRequested) { MapInputWithControlMessage <TMapInput> stopMessage = new MapInputWithControlMessage <TMapInput>(MapControlMessage.Stop); _dataAndControlMessageSender.Send(stopMessage); } } catch (OperationCanceledException e) { Logger.Log(Level.Warning, "Received OperationCanceledException in UpdateTaskHost with message: {0}.", e.Message); } catch (IOException e) { Logger.Log(Level.Error, "Received IOException in UpdateTaskHost with message: {0}.", e.Message); if (!_cancellationSource.IsCancellationRequested) { throw new IMRUTaskGroupCommunicationException(TaskManager.TaskGroupCommunicationError); } } catch (TcpClientConnectionException e) { Logger.Log(Level.Error, "Received TcpClientConnectionException in UpdateTaskHost with message: {0}.", e.Message); if (!_cancellationSource.IsCancellationRequested) { throw new IMRUTaskGroupCommunicationException(TaskManager.TaskGroupCommunicationError); } } _resultHandler.Dispose(); _taskCloseCoordinator.SignalTaskStopped(); Logger.Log(Level.Info, "UpdateTaskHost returned with cancellation token {0}.", _cancellationSource.IsCancellationRequested); return(null); }