private IMRUDriver(IPartitionedInputDataSet dataSet, [Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs, ConfigurationManager configurationManager, IEvaluatorRequestor evaluatorRequestor, [Parameter(typeof(CoresPerMapper))] int coresPerMapper, [Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask, [Parameter(typeof(MemoryPerMapper))] int memoryPerMapper, [Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask, [Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction, [Parameter(typeof(MaxRetryNumberInRecovery))] int maxRetryNumberInRecovery, [Parameter(typeof(InvokeGC))] bool invokeGC, IGroupCommDriver groupCommDriver, INameServer nameServer) { _configurationManager = configurationManager; _groupCommDriver = groupCommDriver; _nameServer = nameServer; _perMapperConfigs = perMapperConfigs; _totalMappers = dataSet.Count; _invokeGC = invokeGC; _maxRetryNumberForFaultTolerant = maxRetryNumberInRecovery > 0 ? maxRetryNumberInRecovery : DefaultMaxNumberOfRetryInRecovery; _contextManager = new ActiveContextManager(_totalMappers + 1); _contextManager.Subscribe(this); var updateSpec = new EvaluatorSpecification(memoryForUpdateTask, coresForUpdateTask); var mapperSpec = new EvaluatorSpecification(memoryPerMapper, coresPerMapper); var allowedFailedEvaluators = (int)(failedEvaluatorsFraction * _totalMappers); _evaluatorManager = new EvaluatorManager(_totalMappers + 1, allowedFailedEvaluators, evaluatorRequestor, updateSpec, mapperSpec); _systemState = new SystemStateMachine(); _serviceAndContextConfigurationProvider = new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput, TPartitionType>(dataSet, configurationManager); var msg = string.Format(CultureInfo.InvariantCulture, "map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}, maxRetry {4}, allowedFailedEvaluators {5}.", memoryPerMapper, memoryForUpdateTask, coresPerMapper, coresForUpdateTask, _maxRetryNumberForFaultTolerant, allowedFailedEvaluators); Logger.Log(Level.Info, msg); }
private IMRUDriver(IPartitionedInputDataSet dataSet, [Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs, ConfigurationManager configurationManager, IEvaluatorRequestor evaluatorRequestor, [Parameter(typeof(CoresPerMapper))] int coresPerMapper, [Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask, [Parameter(typeof(MemoryPerMapper))] int memoryPerMapper, [Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask, [Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction, [Parameter(typeof(InvokeGC))] bool invokeGC, IGroupCommDriver groupCommDriver) { _dataSet = dataSet; _configurationManager = configurationManager; _evaluatorRequestor = evaluatorRequestor; _groupCommDriver = groupCommDriver; _coresPerMapper = coresPerMapper; _coresForUpdateTask = coresForUpdateTask; _memoryPerMapper = memoryPerMapper; _memoryForUpdateTask = memoryForUpdateTask; _perMapperConfigs = perMapperConfigs; _completedTasks = new ConcurrentBag <ICompletedTask>(); _allowedFailedEvaluators = (int)(failedEvaluatorsFraction * dataSet.Count); _invokeGC = invokeGC; AddGroupCommunicationOperators(); _groupCommTaskStarter = new TaskStarter(_groupCommDriver, _dataSet.Count + 1); _taskIdStack = new ConcurrentStack <string>(); _perMapperConfiguration = new ConcurrentStack <IConfiguration>(); _partitionDescriptorStack = new Stack <IPartitionDescriptor>(); ConstructTaskIdAndPartitionDescriptorStack(); _serviceAndContextConfigurationProvider = new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput>(dataSet.Count + 1, groupCommDriver, _configurationManager, _partitionDescriptorStack); var msg = string.Format("map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}", _memoryPerMapper, _memoryForUpdateTask, _coresPerMapper, _coresForUpdateTask); Logger.Log(Level.Info, msg); }