/// <summary>
/// Performs IMRU iterations on update side
/// </summary>
/// <returns></returns>
protected override byte[] TaskBody(byte[] memento)
{
UpdateResult <TMapInput, TResult> updateResult = null;
try
{
updateResult = _updateTask.Initialize();
}
catch (Exception e)
{
HandleTaskAppException(e);
}
while (!_cancellationSource.IsCancellationRequested && updateResult.HasMapInput)
{
using (
var message = new MapInputWithControlMessage <TMapInput>(updateResult.MapInput,
MapControlMessage.AnotherRound))
{
_dataAndControlMessageSender.Send(message);
}
if (_invokeGc)
{
Logger.Log(Level.Verbose, "Calling Garbage Collector");
GC.Collect();
GC.WaitForPendingFinalizers();
}
var input = _dataReceiver.Reduce(_cancellationSource);
try
{
updateResult = _updateTask.Update(input);
if (updateResult.HasResult)
{
_resultHandler.HandleResult(updateResult.Result);
_done = true;
}
}
catch (Exception e)
{
HandleTaskAppException(e);
}
}
if (!_cancellationSource.IsCancellationRequested)
{
MapInputWithControlMessage <TMapInput> stopMessage =
new MapInputWithControlMessage <TMapInput>(MapControlMessage.Stop);
_dataAndControlMessageSender.Send(stopMessage);
}
if (_done)
{
return(ByteUtilities.StringToByteArrays(TaskManager.UpdateTaskCompleted));
}
return(null);
}
public void TestBroadcastOperatorWithDefaultCodec()
{
INameServer nameServer = NameServerTests.BuildNameServer();
string groupName = "group1";
string operatorName = "broadcast";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 10;
int value = 1337;
int fanOut = 3;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddBroadcast(operatorName, masterTaskId)
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName);
IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
sender.Send(value);
Assert.Equal(value, receiver1.Receive());
Assert.Equal(value, receiver2.Receive());
}
public byte[] Call(byte[] memento)
{
int[] intArr = new int[_arraySize];
for (int i = 1; i <= _numIters; i++)
{
for (int j = 0; j < _arraySize; j++)
{
intArr[j] = i;
}
_broadcastSender.Send(intArr);
int[] sum = _sumReducer.Reduce();
Logger.Log(Level.Info, "Received sum: {0} on iteration: {1} with array length: {2}", sum[0], i, sum.Length);
int expected = TriangleNumber(i) * _numReduceSenders;
for (int j = 0; j < intArr.Length; j++)
{
if (sum[j] != TriangleNumber(i) * _numReduceSenders)
{
throw new Exception("Expected " + expected + " but got " + sum);
}
}
}
return(null);
}
/// <summary>
/// Performs IMRU iterations on update side
/// </summary>
/// <param name="memento"></param>
/// <returns></returns>
public byte[] Call(byte[] memento)
{
var updateResult = _updateTask.Initialize();
int iterNo = 0;
while (updateResult.HasMapInput && !_taskCloseCoordinator.ShouldCloseTask())
{
iterNo++;
using (
var message = new MapInputWithControlMessage <TMapInput>(updateResult.MapInput,
MapControlMessage.AnotherRound))
{
_dataAndControlMessageSender.Send(message);
}
var input = _dataReceiver.Reduce();
if (_invokeGC)
{
Logger.Log(Level.Verbose, "Calling Garbage Collector");
GC.Collect();
GC.WaitForPendingFinalizers();
}
updateResult = _updateTask.Update(input);
if (updateResult.HasResult)
{
_resultHandler.HandleResult(updateResult.Result);
}
}
if (!_taskCloseCoordinator.ShouldCloseTask())
{
MapInputWithControlMessage <TMapInput> stopMessage =
new MapInputWithControlMessage <TMapInput>(MapControlMessage.Stop);
_dataAndControlMessageSender.Send(stopMessage);
}
_resultHandler.Dispose();
_taskCloseCoordinator.SignalTaskStopped();
return(null);
}
public void TestBroadcastReduceOperators()
{
string groupName = "group1";
string broadcastOperatorName = "broadcast";
string reduceOperatorName = "reduce";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 3;
int fanOut = 2;
var groupCommunicationDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
ICommunicationGroupDriver commGroup = groupCommunicationDriver.DefaultGroup
.AddBroadcast <int>(
broadcastOperatorName,
masterTaskId,
TopologyTypes.Flat,
GetDefaultDataConverterConfig())
.AddReduce <int>(
reduceOperatorName,
masterTaskId,
TopologyTypes.Flat,
GetDefaultDataConverterConfig(),
GetDefaultReduceFuncConfig())
.Build();
var commGroups = CommGroupClients(groupName, numTasks, groupCommunicationDriver, commGroup, GetDefaultCodecConfig());
// for master task
IBroadcastSender <int> broadcastSender = commGroups[0].GetBroadcastSender <int>(broadcastOperatorName);
IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver1 = commGroups[1].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver2 = commGroups[2].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName);
for (int j = 1; j <= 10; j++)
{
broadcastSender.Send(j);
int n1 = broadcastReceiver1.Receive();
int n2 = broadcastReceiver2.Receive();
Assert.Equal(j, n1);
Assert.Equal(j, n2);
int triangleNum1 = TriangleNumber(n1);
triangleNumberSender1.Send(triangleNum1);
int triangleNum2 = TriangleNumber(n2);
triangleNumberSender2.Send(triangleNum2);
int sum = sumReducer.Reduce();
int expected = TriangleNumber(j) * (numTasks - 1);
Assert.Equal(sum, expected);
}
}
public void TestBroadcastOperator2()
{
string groupName = "group1";
string operatorName = "broadcast";
string driverId = "driverId";
string masterTaskId = "task0";
int numTasks = 3;
int value1 = 1337;
int value2 = 42;
int value3 = 99;
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddBroadcast(operatorName, masterTaskId)
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName);
IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName);
Assert.IsNotNull(sender);
Assert.IsNotNull(receiver1);
Assert.IsNotNull(receiver2);
sender.Send(value1);
Assert.AreEqual(value1, receiver1.Receive());
Assert.AreEqual(value1, receiver2.Receive());
sender.Send(value2);
Assert.AreEqual(value2, receiver1.Receive());
Assert.AreEqual(value2, receiver2.Receive());
sender.Send(value3);
Assert.AreEqual(value3, receiver1.Receive());
Assert.AreEqual(value3, receiver2.Receive());
}
public byte[] Call(byte[] memento)
{
_groupCommClient.Initialize();
int[] intArr = new int[_arraySize];
for (int j = 0; j < _arraySize; j++)
{
intArr[j] = j;
}
Stopwatch broadcastTime = new Stopwatch();
Stopwatch reduceTime = new Stopwatch();
for (int i = 1; i <= _numIters; i++)
{
intArr[0] = i;
if (i == 2)
{
broadcastTime.Reset();
reduceTime.Reset();
}
broadcastTime.Start();
_broadcastSender.Send(intArr);
broadcastTime.Stop();
reduceTime.Start();
int[] sum = _sumReducer.Reduce();
reduceTime.Stop();
Logger.Log(Level.Info, "Received sum: {0} on iteration: {1} with array length: {2}", sum[0], i,
sum.Length);
int expected = TriangleNumber(i) * _numReduceSenders;
if (sum[0] != TriangleNumber(i) * _numReduceSenders)
{
throw new Exception("Expected " + expected + " but got " + sum[0]);
}
if (i >= 2)
{
var msg = string.Format("Average time (milliseconds) taken for broadcast: {0} and reduce: {1}",
broadcastTime.ElapsedMilliseconds / ((double)(i - 1)),
reduceTime.ElapsedMilliseconds / ((double)(i - 1)));
Logger.Log(Level.Info, msg);
}
}
return(null);
}
public byte[] Call(byte[] memento)
{
_groupCommClient.Initialize();
Stopwatch broadcastTime = new Stopwatch();
Stopwatch reduceTime = new Stopwatch();
for (int i = 1; i <= _numIters; i++)
{
if (i == 2)
{
broadcastTime.Reset();
reduceTime.Reset();
}
broadcastTime.Start();
// Each slave task calculates the nth triangle number
_broadcastSender.Send(i);
broadcastTime.Stop();
reduceTime.Start();
// Sum up all of the calculated triangle numbers
int sum = _sumReducer.Reduce();
reduceTime.Stop();
Logger.Log(Level.Info, "Received sum: {0} on iteration: {1}", sum, i);
int expected = TriangleNumber(i) * _numReduceSenders;
if (sum != TriangleNumber(i) * _numReduceSenders)
{
throw new Exception("Expected " + expected + " but got " + sum);
}
if (i >= 2)
{
var msg = string.Format("Average time (milliseconds) taken for broadcast: {0} and reduce: {1}",
broadcastTime.ElapsedMilliseconds / ((double)(i - 1)),
reduceTime.ElapsedMilliseconds / ((double)(i - 1)));
Logger.Log(Level.Info, msg);
}
}
return(null);
}
public byte[] Call(byte[] memento)
{
for (int i = 1; i <= _numIters; i++)
{
// Each slave task calculates the nth triangle number
_broadcastSender.Send(i);
// Sum up all of the calculated triangle numbers
int sum = _sumReducer.Reduce();
_logger.Log(Level.Info, "Received sum: {0} on iteration: {1}", sum, i);
int expected = TriangleNumber(i) * _numReduceSenders;
if (sum != TriangleNumber(i) * _numReduceSenders)
{
throw new Exception("Expected " + expected + " but got " + sum);
}
}
return(null);
}
public void TestBroadcastReduceOperators()
{
string groupName = "group1";
string broadcastOperatorName = "broadcast";
string reduceOperatorName = "reduce";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 10;
int fanOut = 3;
var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup
.AddBroadcast <int>(
broadcastOperatorName,
masterTaskId,
TopologyTypes.Tree,
GetDefaultDataConverterConfig())
.AddReduce <int>(
reduceOperatorName,
masterTaskId,
TopologyTypes.Tree,
GetDefaultDataConverterConfig(),
GetDefaultReduceFuncConfig())
.Build();
var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
// for master task
IBroadcastSender <int> broadcastSender = commGroups[0].GetBroadcastSender <int>(broadcastOperatorName);
IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver1 = commGroups[1].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver2 = commGroups[2].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver3 = commGroups[3].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender3 = commGroups[3].GetReduceSender <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver4 = commGroups[4].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender4 = commGroups[4].GetReduceSender <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver5 = commGroups[5].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender5 = commGroups[5].GetReduceSender <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver6 = commGroups[6].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender6 = commGroups[6].GetReduceSender <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver7 = commGroups[7].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender7 = commGroups[7].GetReduceSender <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver8 = commGroups[8].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender8 = commGroups[8].GetReduceSender <int>(reduceOperatorName);
IBroadcastReceiver <int> broadcastReceiver9 = commGroups[9].GetBroadcastReceiver <int>(broadcastOperatorName);
IReduceSender <int> triangleNumberSender9 = commGroups[9].GetReduceSender <int>(reduceOperatorName);
for (int i = 1; i <= 10; i++)
{
broadcastSender.Send(i);
int n1 = broadcastReceiver1.Receive();
int n2 = broadcastReceiver2.Receive();
int n3 = broadcastReceiver3.Receive();
int n4 = broadcastReceiver4.Receive();
int n5 = broadcastReceiver5.Receive();
int n6 = broadcastReceiver6.Receive();
int n7 = broadcastReceiver7.Receive();
int n8 = broadcastReceiver8.Receive();
int n9 = broadcastReceiver9.Receive();
Assert.Equal(i, n1);
Assert.Equal(i, n2);
Assert.Equal(i, n3);
Assert.Equal(i, n4);
Assert.Equal(i, n5);
Assert.Equal(i, n6);
Assert.Equal(i, n7);
Assert.Equal(i, n8);
Assert.Equal(i, n9);
int triangleNum9 = GroupCommunicationTests.TriangleNumber(n9);
triangleNumberSender9.Send(triangleNum9);
int triangleNum8 = GroupCommunicationTests.TriangleNumber(n8);
triangleNumberSender8.Send(triangleNum8);
int triangleNum7 = GroupCommunicationTests.TriangleNumber(n7);
triangleNumberSender7.Send(triangleNum7);
int triangleNum6 = GroupCommunicationTests.TriangleNumber(n6);
triangleNumberSender6.Send(triangleNum6);
int triangleNum5 = GroupCommunicationTests.TriangleNumber(n5);
triangleNumberSender5.Send(triangleNum5);
int triangleNum4 = GroupCommunicationTests.TriangleNumber(n4);
triangleNumberSender4.Send(triangleNum4);
int triangleNum3 = GroupCommunicationTests.TriangleNumber(n3);
triangleNumberSender3.Send(triangleNum3);
int triangleNum2 = GroupCommunicationTests.TriangleNumber(n2);
triangleNumberSender2.Send(triangleNum2);
int triangleNum1 = GroupCommunicationTests.TriangleNumber(n1);
triangleNumberSender1.Send(triangleNum1);
int sum = sumReducer.Reduce();
int expected = GroupCommunicationTests.TriangleNumber(i) * (numTasks - 1);
Assert.Equal(sum, expected);
}
}
public void TestBroadcastOperator()
{
string groupName = "group1";
string operatorName = "broadcast";
string driverId = "driverId";
string masterTaskId = "task0";
int numTasks = 10;
int value1 = 1337;
int value2 = 42;
int value3 = 99;
int fanOut = 3;
var groupCommDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup
.AddBroadcast <int>(operatorName, masterTaskId, TopologyTypes.Tree, GetDefaultDataConverterConfig(), GetDefaultReduceFuncConfig())
.Build();
var commGroups = GroupCommunicationTests.CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName);
IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver3 = commGroups[3].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver4 = commGroups[4].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver5 = commGroups[5].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver6 = commGroups[6].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver7 = commGroups[7].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver8 = commGroups[8].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver9 = commGroups[9].GetBroadcastReceiver <int>(operatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
Assert.NotNull(receiver3);
Assert.NotNull(receiver4);
Assert.NotNull(receiver5);
Assert.NotNull(receiver6);
Assert.NotNull(receiver7);
Assert.NotNull(receiver8);
Assert.NotNull(receiver9);
sender.Send(value1);
Assert.Equal(value1, receiver1.Receive());
Assert.Equal(value1, receiver2.Receive());
Assert.Equal(value1, receiver3.Receive());
Assert.Equal(value1, receiver4.Receive());
Assert.Equal(value1, receiver5.Receive());
Assert.Equal(value1, receiver6.Receive());
Assert.Equal(value1, receiver7.Receive());
Assert.Equal(value1, receiver8.Receive());
Assert.Equal(value1, receiver9.Receive());
sender.Send(value2);
Assert.Equal(value2, receiver1.Receive());
Assert.Equal(value2, receiver2.Receive());
Assert.Equal(value2, receiver3.Receive());
Assert.Equal(value2, receiver4.Receive());
Assert.Equal(value2, receiver5.Receive());
Assert.Equal(value2, receiver6.Receive());
Assert.Equal(value2, receiver7.Receive());
Assert.Equal(value2, receiver8.Receive());
Assert.Equal(value2, receiver9.Receive());
sender.Send(value3);
Assert.Equal(value3, receiver1.Receive());
Assert.Equal(value3, receiver2.Receive());
Assert.Equal(value3, receiver3.Receive());
Assert.Equal(value3, receiver4.Receive());
Assert.Equal(value3, receiver5.Receive());
Assert.Equal(value3, receiver6.Receive());
Assert.Equal(value3, receiver7.Receive());
Assert.Equal(value3, receiver8.Receive());
Assert.Equal(value3, receiver9.Receive());
}
/// <summary>
/// Performs IMRU iterations on update side
/// </summary>
/// <param name="memento"></param>
/// <returns></returns>
public byte[] Call(byte[] memento)
{
var updateResult = _updateTask.Initialize();
int iterNo = 0;
try
{
while (updateResult.HasMapInput && !_cancellationSource.IsCancellationRequested)
{
iterNo++;
using (
var message = new MapInputWithControlMessage <TMapInput>(updateResult.MapInput,
MapControlMessage.AnotherRound))
{
_dataAndControlMessageSender.Send(message);
}
var input = _dataReceiver.Reduce(_cancellationSource);
if (_invokeGC)
{
Logger.Log(Level.Verbose, "Calling Garbage Collector");
GC.Collect();
GC.WaitForPendingFinalizers();
}
updateResult = _updateTask.Update(input);
if (updateResult.HasResult)
{
_resultHandler.HandleResult(updateResult.Result);
}
}
if (!_cancellationSource.IsCancellationRequested)
{
MapInputWithControlMessage <TMapInput> stopMessage =
new MapInputWithControlMessage <TMapInput>(MapControlMessage.Stop);
_dataAndControlMessageSender.Send(stopMessage);
}
}
catch (OperationCanceledException e)
{
Logger.Log(Level.Warning,
"Received OperationCanceledException in UpdateTaskHost with message: {0}.",
e.Message);
}
catch (IOException e)
{
Logger.Log(Level.Error, "Received IOException in UpdateTaskHost with message: {0}.", e.Message);
if (!_cancellationSource.IsCancellationRequested)
{
throw new IMRUTaskGroupCommunicationException(TaskManager.TaskGroupCommunicationError);
}
}
catch (TcpClientConnectionException e)
{
Logger.Log(Level.Error,
"Received TcpClientConnectionException in UpdateTaskHost with message: {0}.",
e.Message);
if (!_cancellationSource.IsCancellationRequested)
{
throw new IMRUTaskGroupCommunicationException(TaskManager.TaskGroupCommunicationError);
}
}
_resultHandler.Dispose();
_taskCloseCoordinator.SignalTaskStopped();
Logger.Log(Level.Info, "UpdateTaskHost returned with cancellation token {0}.", _cancellationSource.IsCancellationRequested);
return(null);
}