public PipelinedBroadcastReduceDriver(
[Parameter(typeof(GroupTestConfig.NumEvaluators))] int numEvaluators,
[Parameter(typeof(GroupTestConfig.NumIterations))] int numIterations,
[Parameter(typeof(GroupTestConfig.StartingPort))] int startingPort,
[Parameter(typeof(GroupTestConfig.PortRange))] int portRange,
[Parameter(typeof(GroupTestConfig.ChunkSize))] int chunkSize,
[Parameter(typeof(GroupTestConfig.ArraySize))] int arraySize,
GroupCommDriver groupCommDriver,
IEvaluatorRequestor evaluatorRequestor)
{
Logger.Log(Level.Info, "entering the driver code " + chunkSize);
_numEvaluators = numEvaluators;
_numIterations = numIterations;
_arraySize = arraySize;
_evaluatorRequestor = evaluatorRequestor;
_tcpPortProviderConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindNamedParameter <TcpPortRangeStart, int>(GenericType <TcpPortRangeStart> .Class,
startingPort.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter <TcpPortRangeCount, int>(GenericType <TcpPortRangeCount> .Class,
portRange.ToString(CultureInfo.InvariantCulture))
.Build();
_codecConfig = StreamingCodecConfiguration <int[]> .Conf
.Set(StreamingCodecConfiguration <int[]> .Codec, GenericType <IntArrayStreamingCodec> .Class)
.Build();
var reduceFunctionConfig = ReduceFunctionConfiguration <int[]> .Conf
.Set(ReduceFunctionConfiguration <int[]> .ReduceFunction, GenericType <ArraySumFunction> .Class)
.Build();
var dataConverterConfig = TangFactory.GetTang().NewConfigurationBuilder(
PipelineDataConverterConfiguration <int[]> .Conf
.Set(PipelineDataConverterConfiguration <int[]> .DataConverter,
GenericType <PipelineIntDataConverter> .Class)
.Build())
.BindNamedParameter <GroupTestConfig.ChunkSize, int>(
GenericType <GroupTestConfig.ChunkSize> .Class,
chunkSize.ToString(CultureInfo.InvariantCulture))
.Build();
_groupCommDriver = groupCommDriver;
_commGroup = _groupCommDriver.DefaultGroup
.AddBroadcast <int[]>(
GroupTestConstants.BroadcastOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
dataConverterConfig)
.AddReduce <int[]>(
GroupTestConstants.ReduceOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
reduceFunctionConfig,
dataConverterConfig)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, numEvaluators);
}
public void TestBroadcastOperatorWithDefaultCodec()
{
INameServer nameServer = NameServerTests.BuildNameServer();
string groupName = "group1";
string operatorName = "broadcast";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 10;
int value = 1337;
int fanOut = 3;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddBroadcast(operatorName, masterTaskId)
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName);
IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
sender.Send(value);
Assert.Equal(value, receiver1.Receive());
Assert.Equal(value, receiver2.Receive());
}
public void TestReduceOperator()
{
string groupName = "group1";
string operatorName = "reduce";
int numTasks = 4;
string driverId = "driverid";
string masterTaskId = "task0";
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddReduce <int>(operatorName, "task0", TopologyTypes.Flat, GetDefaultDataConverterConfig(), GetDefaultReduceFuncConfig())
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IReduceReceiver <int> receiver = commGroups[0].GetReduceReceiver <int>(operatorName);
IReduceSender <int> sender1 = commGroups[1].GetReduceSender <int>(operatorName);
IReduceSender <int> sender2 = commGroups[2].GetReduceSender <int>(operatorName);
IReduceSender <int> sender3 = commGroups[3].GetReduceSender <int>(operatorName);
Assert.NotNull(receiver);
Assert.NotNull(sender1);
Assert.NotNull(sender2);
Assert.NotNull(sender3);
sender3.Send(5);
sender1.Send(1);
sender2.Send(3);
Assert.Equal(9, receiver.Reduce());
}
private KMeansDriverHandlers(
[Parameter(typeof(NumPartitions))] int numPartitions,
GroupCommDriver groupCommDriver,
IEvaluatorRequestor evaluatorRequestor,
CommandLineArguments arguments)
{
_executionDirectory = Path.Combine(Directory.GetCurrentDirectory(), Constants.KMeansExecutionBaseDirectory, Guid.NewGuid().ToString("N").Substring(0, 4));
string dataFile = arguments.Arguments.First();
DataVector.ShuffleDataAndGetInitialCentriods(
dataFile,
numPartitions,
_clustersNumber,
_executionDirectory);
_totalEvaluators = numPartitions + 1;
_groupCommDriver = groupCommDriver;
_evaluatorRequestor = evaluatorRequestor;
_centroidCodecConf = CodecToStreamingCodecConfiguration <Centroids> .Conf
.Set(CodecToStreamingCodecConfiguration <Centroids> .Codec, GenericType <CentroidsCodec> .Class)
.Build();
IConfiguration dataConverterConfig1 = PipelineDataConverterConfiguration <Centroids> .Conf
.Set(PipelineDataConverterConfiguration <Centroids> .DataConverter, GenericType <DefaultPipelineDataConverter <Centroids> > .Class)
.Build();
_controlMessageCodecConf = CodecToStreamingCodecConfiguration <ControlMessage> .Conf
.Set(CodecToStreamingCodecConfiguration <ControlMessage> .Codec, GenericType <ControlMessageCodec> .Class)
.Build();
IConfiguration dataConverterConfig2 = PipelineDataConverterConfiguration <ControlMessage> .Conf
.Set(PipelineDataConverterConfiguration <ControlMessage> .DataConverter, GenericType <DefaultPipelineDataConverter <ControlMessage> > .Class)
.Build();
_processedResultsCodecConf = CodecToStreamingCodecConfiguration <ProcessedResults> .Conf
.Set(CodecToStreamingCodecConfiguration <ProcessedResults> .Codec, GenericType <ProcessedResultsCodec> .Class)
.Build();
IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <ProcessedResults> .Conf
.Set(ReduceFunctionConfiguration <ProcessedResults> .ReduceFunction, GenericType <KMeansMasterTask.AggregateMeans> .Class)
.Build();
IConfiguration dataConverterConfig3 = PipelineDataConverterConfiguration <ProcessedResults> .Conf
.Set(PipelineDataConverterConfiguration <ProcessedResults> .DataConverter, GenericType <DefaultPipelineDataConverter <ProcessedResults> > .Class)
.Build();
_commGroup = _groupCommDriver.DefaultGroup
.AddBroadcast <Centroids>(Constants.CentroidsBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig1)
.AddBroadcast <ControlMessage>(Constants.ControlMessageBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig2)
.AddReduce <ProcessedResults>(Constants.MeansReduceOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, reduceFunctionConfig, dataConverterConfig3)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, _totalEvaluators);
}
public KMeansDriverHandlers([Parameter(typeof(NumPartitions))] int numPartitions, GroupCommDriver groupCommDriver)
{
Identifier = "KMeansDriverId";
_executionDirectory = Path.Combine(Directory.GetCurrentDirectory(), Constants.KMeansExecutionBaseDirectory, Guid.NewGuid().ToString("N").Substring(0, 4));
ISet <string> arguments = ClrHandlerHelper.GetCommandLineArguments();
string dataFile = arguments.Single(a => a.StartsWith("DataFile", StringComparison.Ordinal)).Split(':')[1];
DataVector.ShuffleDataAndGetInitialCentriods(
Path.Combine(Directory.GetCurrentDirectory(), "reef", "global", dataFile),
numPartitions,
_clustersNumber,
_executionDirectory);
_totalEvaluators = numPartitions + 1;
_groupCommDriver = groupCommDriver;
_centroidCodecConf = CodecToStreamingCodecConfiguration <Centroids> .Conf
.Set(CodecConfiguration <Centroids> .Codec, GenericType <CentroidsCodec> .Class)
.Build();
IConfiguration dataConverterConfig1 = PipelineDataConverterConfiguration <Centroids> .Conf
.Set(PipelineDataConverterConfiguration <Centroids> .DataConverter, GenericType <DefaultPipelineDataConverter <Centroids> > .Class)
.Build();
_controlMessageCodecConf = CodecToStreamingCodecConfiguration <ControlMessage> .Conf
.Set(CodecConfiguration <ControlMessage> .Codec, GenericType <ControlMessageCodec> .Class)
.Build();
IConfiguration dataConverterConfig2 = PipelineDataConverterConfiguration <ControlMessage> .Conf
.Set(PipelineDataConverterConfiguration <ControlMessage> .DataConverter, GenericType <DefaultPipelineDataConverter <ControlMessage> > .Class)
.Build();
_processedResultsCodecConf = CodecToStreamingCodecConfiguration <ProcessedResults> .Conf
.Set(CodecConfiguration <ProcessedResults> .Codec, GenericType <ProcessedResultsCodec> .Class)
.Build();
IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <ProcessedResults> .Conf
.Set(ReduceFunctionConfiguration <ProcessedResults> .ReduceFunction, GenericType <KMeansMasterTask.AggregateMeans> .Class)
.Build();
IConfiguration dataConverterConfig3 = PipelineDataConverterConfiguration <ProcessedResults> .Conf
.Set(PipelineDataConverterConfiguration <ProcessedResults> .DataConverter, GenericType <DefaultPipelineDataConverter <ProcessedResults> > .Class)
.Build();
_commGroup = _groupCommDriver.DefaultGroup
.AddBroadcast <Centroids>(Constants.CentroidsBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig1)
.AddBroadcast <ControlMessage>(Constants.ControlMessageBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig2)
.AddReduce <ProcessedResults>(Constants.MeansReduceOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, reduceFunctionConfig, dataConverterConfig3)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, _totalEvaluators);
CreateClassHierarchy();
}
/// <summary>
/// Create new TaskStarter.
/// After adding the correct number of tasks to the TaskStarter, the
/// Tasks will be started on their given active context.
/// </summary>
/// <param name="groupCommDriver">The IGroupCommuDriver for the Group Communication tasks</param>
/// <param name="numTasks">The number of Tasks that need to be added before
/// the Tasks will be started. </param>
public TaskStarter(IGroupCommDriver groupCommDriver, int numTasks)
{
LOGGER.Log(Level.Verbose, "Creating TaskStarter");
_masterTaskId = groupCommDriver.MasterTaskId;
_numTasks = numTasks;
_tasksAdded = 0;
_lock = new object();
_groupCommDriver = groupCommDriver;
_taskTuples = new List <Tuple <string, IConfiguration, IActiveContext> >();
}
/// <summary>
/// Create new TaskStarter.
/// After adding the correct number of tasks to the TaskStarter, the
/// Tasks will be started on their given active context.
/// </summary>
/// <param name="groupCommDriver">The IGroupCommuDriver for the Group Communication tasks</param>
/// <param name="numTasks">The number of Tasks that need to be added before
/// the Tasks will be started. </param>
public TaskStarter(IGroupCommDriver groupCommDriver, int numTasks)
{
LOGGER.Log(Level.Verbose, "Creating TaskStarter");
_masterTaskId = groupCommDriver.MasterTaskId;
_numTasks = numTasks;
_tasksAdded = 0;
_lock = new object();
_groupCommDriver = groupCommDriver;
_taskTuples = new List<Tuple<string, IConfiguration, IActiveContext>>();
}
public void TestScatterOperator2()
{
string groupName = "group1";
string operatorName = "scatter";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 5;
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddScatter <int>(operatorName, masterTaskId, TopologyTypes.Flat, GetDefaultDataConverterConfig(), GetDefaultReduceFuncConfig())
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(operatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(operatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
Assert.NotNull(receiver3);
Assert.NotNull(receiver4);
List <int> data = new List <int> {
1, 2, 3, 4, 5, 6, 7, 8
};
sender.Send(data);
var data1 = receiver1.Receive();
Assert.Equal(1, data1.First());
Assert.Equal(2, data1.Last());
var data2 = receiver2.Receive();
Assert.Equal(3, data2.First());
Assert.Equal(4, data2.Last());
var data3 = receiver3.Receive();
Assert.Equal(5, data3.First());
Assert.Equal(6, data3.Last());
var data4 = receiver4.Receive();
Assert.Equal(7, data4.First());
Assert.Equal(8, data4.Last());
}
public void TestScatterOperator4()
{
string groupName = "group1";
string operatorName = "scatter";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 4;
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddScatter <int>(operatorName, masterTaskId, TopologyTypes.Flat, GetDefaulDataConverterConfig())
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(operatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(operatorName);
Assert.IsNotNull(sender);
Assert.IsNotNull(receiver1);
Assert.IsNotNull(receiver2);
Assert.IsNotNull(receiver3);
List <int> data = new List <int> {
1, 2, 3, 4, 5, 6, 7, 8
};
List <string> order = new List <string> {
"task3", "task2", "task1"
};
sender.Send(data, order);
var data3 = receiver3.Receive().ToArray();
Assert.AreEqual(1, data3[0]);
Assert.AreEqual(2, data3[1]);
Assert.AreEqual(3, data3[2]);
var data2 = receiver2.Receive().ToArray();
Assert.AreEqual(4, data2[0]);
Assert.AreEqual(5, data2[1]);
Assert.AreEqual(6, data2[2]);
var data1 = receiver1.Receive().ToArray();
Assert.AreEqual(7, data1[0]);
Assert.AreEqual(8, data1[1]);
}
internal ServiceAndContextConfigurationProvider(int numNodes, IGroupCommDriver groupCommDriver,
ConfigurationManager configurationManager, Stack <IPartitionDescriptor> partitionDescriptors)
{
_configurationProvider = new Dictionary <string, ContextAndServiceConfiguration>();
_failedEvaluators = new HashSet <string>();
_submittedEvaluators = new HashSet <string>();
_numNodes = numNodes;
_groupCommDriver = groupCommDriver;
_configurationManager = configurationManager;
_assignedPartitionDescriptors = 0;
_partitionDescriptors = partitionDescriptors;
_lock = new object();
}
public BroadcastReduceDriver(
[Parameter(typeof(GroupTestConfig.NumEvaluators))] int numEvaluators,
[Parameter(typeof(GroupTestConfig.NumIterations))] int numIterations,
[Parameter(typeof(GroupTestConfig.StartingPort))] int startingPort,
[Parameter(typeof(GroupTestConfig.PortRange))] int portRange,
GroupCommDriver groupCommDriver)
{
Identifier = "BroadcastStartHandler";
_numEvaluators = numEvaluators;
_numIterations = numIterations;
_groupCommDriver = groupCommDriver;
_tcpPortProviderConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindNamedParameter <TcpPortRangeStart, int>(GenericType <TcpPortRangeStart> .Class,
startingPort.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter <TcpPortRangeCount, int>(GenericType <TcpPortRangeCount> .Class,
portRange.ToString(CultureInfo.InvariantCulture))
.Build();
_codecConfig = StreamingCodecConfiguration <int> .Conf
.Set(StreamingCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class)
.Build();
IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <int> .Conf
.Set(ReduceFunctionConfiguration <int> .ReduceFunction, GenericType <SumFunction> .Class)
.Build();
IConfiguration dataConverterConfig = PipelineDataConverterConfiguration <int> .Conf
.Set(PipelineDataConverterConfiguration <int> .DataConverter, GenericType <DefaultPipelineDataConverter <int> > .Class)
.Build();
_commGroup = _groupCommDriver.DefaultGroup
.AddBroadcast <int>(
GroupTestConstants.BroadcastOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
dataConverterConfig)
.AddReduce <int>(
GroupTestConstants.ReduceOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
reduceFunctionConfig,
dataConverterConfig)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, numEvaluators);
CreateClassHierarchy();
}
public static IGroupCommDriver GetInstanceOfGroupCommDriver(string driverId, string masterTaskId, string groupName, int fanOut, int numTasks)
{
var c = TangFactory.GetTang().NewConfigurationBuilder()
.BindStringNamedParam <GroupCommConfigurationOptions.DriverId>(driverId)
.BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(masterTaskId)
.BindStringNamedParam <GroupCommConfigurationOptions.GroupName>(groupName)
.BindIntNamedParam <GroupCommConfigurationOptions.FanOut>(fanOut.ToString())
.BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>(numTasks.ToString())
.BindImplementation(GenericType <IConfigurationSerializer> .Class, GenericType <AvroConfigurationSerializer> .Class)
.Build();
IGroupCommDriver groupCommDriver = TangFactory.GetTang().NewInjector(c).GetInstance <GroupCommDriver>();
return(groupCommDriver);
}
private IMRUDriver(IPartitionedInputDataSet dataSet,
[Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs,
ConfigurationManager configurationManager,
IEvaluatorRequestor evaluatorRequestor,
[Parameter(typeof(CoresPerMapper))] int coresPerMapper,
[Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask,
[Parameter(typeof(MemoryPerMapper))] int memoryPerMapper,
[Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask,
[Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction,
[Parameter(typeof(MaxRetryNumberInRecovery))] int maxRetryNumberInRecovery,
[Parameter(typeof(InvokeGC))] bool invokeGC,
IGroupCommDriver groupCommDriver,
INameServer nameServer)
{
_configurationManager = configurationManager;
_groupCommDriver = groupCommDriver;
_nameServer = nameServer;
_perMapperConfigs = perMapperConfigs;
_totalMappers = dataSet.Count;
_invokeGC = invokeGC;
_maxRetryNumberForFaultTolerant = maxRetryNumberInRecovery > 0 ? maxRetryNumberInRecovery : DefaultMaxNumberOfRetryInRecovery;
_contextManager = new ActiveContextManager(_totalMappers + 1);
_contextManager.Subscribe(this);
var updateSpec = new EvaluatorSpecification(memoryForUpdateTask, coresForUpdateTask);
var mapperSpec = new EvaluatorSpecification(memoryPerMapper, coresPerMapper);
var allowedFailedEvaluators = (int)(failedEvaluatorsFraction * _totalMappers);
_evaluatorManager = new EvaluatorManager(_totalMappers + 1, allowedFailedEvaluators, evaluatorRequestor, updateSpec, mapperSpec);
_systemState = new SystemStateMachine();
_serviceAndContextConfigurationProvider =
new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput, TPartitionType>(dataSet, configurationManager);
var msg =
string.Format(CultureInfo.InvariantCulture, "map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}, maxRetry {4}, allowedFailedEvaluators {5}.",
memoryPerMapper,
memoryForUpdateTask,
coresPerMapper,
coresForUpdateTask,
_maxRetryNumberForFaultTolerant,
allowedFailedEvaluators);
Logger.Log(Level.Info, msg);
}
private IMRUDriver(IPartitionedInputDataSet dataSet,
[Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs,
ConfigurationManager configurationManager,
IEvaluatorRequestor evaluatorRequestor,
[Parameter(typeof(CoresPerMapper))] int coresPerMapper,
[Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask,
[Parameter(typeof(MemoryPerMapper))] int memoryPerMapper,
[Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask,
[Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction,
[Parameter(typeof(InvokeGC))] bool invokeGC,
IGroupCommDriver groupCommDriver)
{
_dataSet = dataSet;
_configurationManager = configurationManager;
_evaluatorRequestor = evaluatorRequestor;
_groupCommDriver = groupCommDriver;
_coresPerMapper = coresPerMapper;
_coresForUpdateTask = coresForUpdateTask;
_memoryPerMapper = memoryPerMapper;
_memoryForUpdateTask = memoryForUpdateTask;
_perMapperConfigs = perMapperConfigs;
_completedTasks = new ConcurrentBag <ICompletedTask>();
_allowedFailedEvaluators = (int)(failedEvaluatorsFraction * dataSet.Count);
_invokeGC = invokeGC;
AddGroupCommunicationOperators();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, _dataSet.Count + 1);
_taskIdStack = new ConcurrentStack <string>();
_perMapperConfiguration = new ConcurrentStack <IConfiguration>();
_partitionDescriptorStack = new Stack <IPartitionDescriptor>();
ConstructTaskIdAndPartitionDescriptorStack();
_serviceAndContextConfigurationProvider =
new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput>(dataSet.Count + 1, groupCommDriver,
_configurationManager, _partitionDescriptorStack);
var msg = string.Format("map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}",
_memoryPerMapper, _memoryForUpdateTask, _coresPerMapper, _coresForUpdateTask);
Logger.Log(Level.Info, msg);
}
public void TestBroadcastOperator2()
{
string groupName = "group1";
string operatorName = "broadcast";
string driverId = "driverId";
string masterTaskId = "task0";
int numTasks = 3;
int value1 = 1337;
int value2 = 42;
int value3 = 99;
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddBroadcast(operatorName, masterTaskId)
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IBroadcastSender <int> sender = commGroups[0].GetBroadcastSender <int>(operatorName);
IBroadcastReceiver <int> receiver1 = commGroups[1].GetBroadcastReceiver <int>(operatorName);
IBroadcastReceiver <int> receiver2 = commGroups[2].GetBroadcastReceiver <int>(operatorName);
Assert.IsNotNull(sender);
Assert.IsNotNull(receiver1);
Assert.IsNotNull(receiver2);
sender.Send(value1);
Assert.AreEqual(value1, receiver1.Receive());
Assert.AreEqual(value1, receiver2.Receive());
sender.Send(value2);
Assert.AreEqual(value2, receiver1.Receive());
Assert.AreEqual(value2, receiver2.Receive());
sender.Send(value3);
Assert.AreEqual(value3, receiver1.Receive());
Assert.AreEqual(value3, receiver2.Receive());
}
public void TestScatterOperatorWithDefaultCodec()
{
string groupName = "group1";
string operatorName = "scatter";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 5;
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
var commGroup = groupCommDriver.DefaultGroup
.AddScatter(operatorName, masterTaskId)
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(operatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(operatorName);
IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(operatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
Assert.NotNull(receiver3);
Assert.NotNull(receiver4);
List <int> data = new List <int> {
1, 2, 3, 4
};
sender.Send(data);
Assert.Equal(1, receiver1.Receive().Single());
Assert.Equal(2, receiver2.Receive().Single());
Assert.Equal(3, receiver3.Receive().Single());
Assert.Equal(4, receiver4.Receive().Single());
}
public ScatterReduceDriver(
[Parameter(typeof(GroupTestConfig.NumEvaluators))] int numEvaluators,
GroupCommDriver groupCommDriver)
{
Identifier = "BroadcastStartHandler";
_numEvaluators = numEvaluators;
_groupCommDriver = groupCommDriver;
_codecConfig = StreamingCodecConfiguration <int> .Conf
.Set(StreamingCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class)
.Build();
IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <int> .Conf
.Set(ReduceFunctionConfiguration <int> .ReduceFunction, GenericType <SumFunction> .Class)
.Build();
IConfiguration dataConverterConfig = PipelineDataConverterConfiguration <int> .Conf
.Set(PipelineDataConverterConfiguration <int> .DataConverter, GenericType <DefaultPipelineDataConverter <int> > .Class)
.Build();
_commGroup = _groupCommDriver.DefaultGroup
.AddScatter <int>(
GroupTestConstants.ScatterOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
dataConverterConfig)
.AddReduce <int>(
GroupTestConstants.ReduceOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
reduceFunctionConfig,
dataConverterConfig)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, numEvaluators);
CreateClassHierarchy();
}
public static List <ICommunicationGroupClient> CommGroupClients(string groupName, int numTasks, IGroupCommDriver groupCommDriver, ICommunicationGroupDriver commGroupDriver, IConfiguration userServiceConfig)
{
List <ICommunicationGroupClient> commGroups = new List <ICommunicationGroupClient>();
IConfiguration serviceConfig = groupCommDriver.GetServiceConfiguration();
serviceConfig = Configurations.Merge(serviceConfig, userServiceConfig);
List <IConfiguration> partialConfigs = new List <IConfiguration>();
for (int i = 0; i < numTasks; i++)
{
string taskId = "task" + i;
IConfiguration partialTaskConfig = TangFactory.GetTang().NewConfigurationBuilder(
TaskConfiguration.ConfigurationModule
.Set(TaskConfiguration.Identifier, taskId)
.Set(TaskConfiguration.Task, GenericType <MyTask> .Class)
.Build())
.Build();
commGroupDriver.AddTask(taskId);
partialConfigs.Add(partialTaskConfig);
}
for (int i = 0; i < numTasks; i++)
{
// get task configuration at driver side
string taskId = "task" + i;
IConfiguration groupCommTaskConfig = groupCommDriver.GetGroupCommTaskConfiguration(taskId);
IConfiguration mergedConf = Configurations.Merge(groupCommTaskConfig, partialConfigs[i], serviceConfig);
var conf = TangFactory.GetTang()
.NewConfigurationBuilder(mergedConf)
.BindNamedParameter(typeof(GroupCommConfigurationOptions.Initialize), "false")
.Build();
IInjector injector = TangFactory.GetTang().NewInjector(conf);
// simulate injection at evaluator side
IGroupCommClient groupCommClient = injector.GetInstance <IGroupCommClient>();
commGroups.Add(groupCommClient.GetCommunicationGroup(groupName));
}
return(commGroups);
}
public void TestScatterReduceOperators()
{
string groupName = "group1";
string scatterOperatorName = "scatter";
string reduceOperatorName = "reduce";
string masterTaskId = "task0";
string driverId = "Driver Id";
int numTasks = 5;
int fanOut = 2;
IGroupCommDriver groupCommDriver = GetInstanceOfGroupCommDriver(driverId, masterTaskId, groupName, fanOut, numTasks);
ICommunicationGroupDriver commGroup = groupCommDriver.DefaultGroup
.AddScatter <int>(
scatterOperatorName,
masterTaskId,
TopologyTypes.Flat,
GetDefaultDataConverterConfig())
.AddReduce <int>(
reduceOperatorName,
masterTaskId,
TopologyTypes.Flat,
GetDefaultReduceFuncConfig(),
GetDefaultDataConverterConfig())
.Build();
List <ICommunicationGroupClient> commGroups = CommGroupClients(groupName, numTasks, groupCommDriver, commGroup, GetDefaultCodecConfig());
IScatterSender <int> sender = commGroups[0].GetScatterSender <int>(scatterOperatorName);
IReduceReceiver <int> sumReducer = commGroups[0].GetReduceReceiver <int>(reduceOperatorName);
IScatterReceiver <int> receiver1 = commGroups[1].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender1 = commGroups[1].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver2 = commGroups[2].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender2 = commGroups[2].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver3 = commGroups[3].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender3 = commGroups[3].GetReduceSender <int>(reduceOperatorName);
IScatterReceiver <int> receiver4 = commGroups[4].GetScatterReceiver <int>(scatterOperatorName);
IReduceSender <int> sumSender4 = commGroups[4].GetReduceSender <int>(reduceOperatorName);
Assert.NotNull(sender);
Assert.NotNull(receiver1);
Assert.NotNull(receiver2);
Assert.NotNull(receiver3);
Assert.NotNull(receiver4);
List <int> data = Enumerable.Range(1, 100).ToList();
List <string> order = new List <string> {
"task4", "task3", "task2", "task1"
};
sender.Send(data, order);
ScatterReceiveReduce(receiver4, sumSender4);
ScatterReceiveReduce(receiver3, sumSender3);
ScatterReceiveReduce(receiver2, sumSender2);
ScatterReceiveReduce(receiver1, sumSender1);
int sum = sumReducer.Reduce();
Assert.Equal(sum, data.Sum());
}
