private IMRUDriver(IPartitionedInputDataSet dataSet,
[Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs,
ConfigurationManager configurationManager,
IEvaluatorRequestor evaluatorRequestor,
[Parameter(typeof(CoresPerMapper))] int coresPerMapper,
[Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask,
[Parameter(typeof(MemoryPerMapper))] int memoryPerMapper,
[Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask,
[Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction,
[Parameter(typeof(MaxRetryNumberInRecovery))] int maxRetryNumberInRecovery,
[Parameter(typeof(InvokeGC))] bool invokeGC,
IGroupCommDriver groupCommDriver,
INameServer nameServer)
{
_configurationManager = configurationManager;
_groupCommDriver = groupCommDriver;
_nameServer = nameServer;
_perMapperConfigs = perMapperConfigs;
_totalMappers = dataSet.Count;
_invokeGC = invokeGC;
_maxRetryNumberForFaultTolerant = maxRetryNumberInRecovery > 0 ? maxRetryNumberInRecovery : DefaultMaxNumberOfRetryInRecovery;
_contextManager = new ActiveContextManager(_totalMappers + 1);
_contextManager.Subscribe(this);
var updateSpec = new EvaluatorSpecification(memoryForUpdateTask, coresForUpdateTask);
var mapperSpec = new EvaluatorSpecification(memoryPerMapper, coresPerMapper);
var allowedFailedEvaluators = (int)(failedEvaluatorsFraction * _totalMappers);
_evaluatorManager = new EvaluatorManager(_totalMappers + 1, allowedFailedEvaluators, evaluatorRequestor, updateSpec, mapperSpec);
_systemState = new SystemStateMachine();
_serviceAndContextConfigurationProvider =
new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput, TPartitionType>(dataSet, configurationManager);
var msg =
string.Format(CultureInfo.InvariantCulture, "map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}, maxRetry {4}, allowedFailedEvaluators {5}.",
memoryPerMapper,
memoryForUpdateTask,
coresPerMapper,
coresForUpdateTask,
_maxRetryNumberForFaultTolerant,
allowedFailedEvaluators);
Logger.Log(Level.Info, msg);
}
private IMRUDriver(IPartitionedInputDataSet dataSet,
[Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs,
ConfigurationManager configurationManager,
IEvaluatorRequestor evaluatorRequestor,
[Parameter(typeof(CoresPerMapper))] int coresPerMapper,
[Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask,
[Parameter(typeof(MemoryPerMapper))] int memoryPerMapper,
[Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask,
[Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction,
[Parameter(typeof(InvokeGC))] bool invokeGC,
IGroupCommDriver groupCommDriver)
{
_dataSet = dataSet;
_configurationManager = configurationManager;
_evaluatorRequestor = evaluatorRequestor;
_groupCommDriver = groupCommDriver;
_coresPerMapper = coresPerMapper;
_coresForUpdateTask = coresForUpdateTask;
_memoryPerMapper = memoryPerMapper;
_memoryForUpdateTask = memoryForUpdateTask;
_perMapperConfigs = perMapperConfigs;
_completedTasks = new ConcurrentBag <ICompletedTask>();
_allowedFailedEvaluators = (int)(failedEvaluatorsFraction * dataSet.Count);
_invokeGC = invokeGC;
AddGroupCommunicationOperators();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, _dataSet.Count + 1);
_taskIdStack = new ConcurrentStack <string>();
_perMapperConfiguration = new ConcurrentStack <IConfiguration>();
_partitionDescriptorStack = new Stack <IPartitionDescriptor>();
ConstructTaskIdAndPartitionDescriptorStack();
_serviceAndContextConfigurationProvider =
new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput>(dataSet.Count + 1, groupCommDriver,
_configurationManager, _partitionDescriptorStack);
var msg = string.Format("map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}",
_memoryPerMapper, _memoryForUpdateTask, _coresPerMapper, _coresForUpdateTask);
Logger.Log(Level.Info, msg);
}