/// <summary>
/// Creates a EvaluatorManager for driver which contains specification for the Evaluators
/// </summary>
/// <param name="totalEvaluators"></param>
/// <param name="allowedNumberOfEvaluatorFailures"></param>
/// <param name="evaluatorRequestor"></param>
/// <param name="updateEvaluatorSpecification"></param>
/// <param name="mapperEvaluatorSpecification"></param>
internal EvaluatorManager(
int totalEvaluators,
int allowedNumberOfEvaluatorFailures,
IEvaluatorRequestor evaluatorRequestor,
EvaluatorSpecification updateEvaluatorSpecification,
EvaluatorSpecification mapperEvaluatorSpecification)
{
_totalExpectedEvaluators = totalEvaluators;
_allowedNumberOfEvaluatorFailures = allowedNumberOfEvaluatorFailures;
_evaluatorRequestor = evaluatorRequestor;
_updateEvaluatorSpecification = updateEvaluatorSpecification;
_mapperEvaluatorSpecification = mapperEvaluatorSpecification;
}
/// <summary>
/// Creates a EvaluatorManager for driver which contains specification for the Evaluators
/// </summary>
/// <param name="totalEvaluators"></param>
/// <param name="allowedNumberOfEvaluatorFailures"></param>
/// <param name="evaluatorRequestor"></param>
/// <param name="updateEvaluatorSpecification"></param>
/// <param name="mapperEvaluatorSpecification"></param>
internal EvaluatorManager(
int totalEvaluators,
int allowedNumberOfEvaluatorFailures,
IEvaluatorRequestor evaluatorRequestor,
EvaluatorSpecification updateEvaluatorSpecification,
EvaluatorSpecification mapperEvaluatorSpecification)
{
_totalExpectedEvaluators = totalEvaluators;
_allowedNumberOfEvaluatorFailures = allowedNumberOfEvaluatorFailures;
_evaluatorRequestor = evaluatorRequestor;
_updateEvaluatorSpecification = updateEvaluatorSpecification;
_mapperEvaluatorSpecification = mapperEvaluatorSpecification;
}
private IMRUDriver(IPartitionedInputDataSet dataSet,
[Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs,
ConfigurationManager configurationManager,
IEvaluatorRequestor evaluatorRequestor,
[Parameter(typeof(CoresPerMapper))] int coresPerMapper,
[Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask,
[Parameter(typeof(MemoryPerMapper))] int memoryPerMapper,
[Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask,
[Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction,
[Parameter(typeof(MaxRetryNumberInRecovery))] int maxRetryNumberInRecovery,
[Parameter(typeof(InvokeGC))] bool invokeGC,
IGroupCommDriver groupCommDriver,
INameServer nameServer)
{
_configurationManager = configurationManager;
_groupCommDriver = groupCommDriver;
_nameServer = nameServer;
_perMapperConfigs = perMapperConfigs;
_totalMappers = dataSet.Count;
_invokeGC = invokeGC;
_maxRetryNumberForFaultTolerant = maxRetryNumberInRecovery > 0 ? maxRetryNumberInRecovery : DefaultMaxNumberOfRetryInRecovery;
_contextManager = new ActiveContextManager(_totalMappers + 1);
_contextManager.Subscribe(this);
var updateSpec = new EvaluatorSpecification(memoryForUpdateTask, coresForUpdateTask);
var mapperSpec = new EvaluatorSpecification(memoryPerMapper, coresPerMapper);
var allowedFailedEvaluators = (int)(failedEvaluatorsFraction * _totalMappers);
_evaluatorManager = new EvaluatorManager(_totalMappers + 1, allowedFailedEvaluators, evaluatorRequestor, updateSpec, mapperSpec);
_systemState = new SystemStateMachine();
_serviceAndContextConfigurationProvider =
new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput, TPartitionType>(dataSet, configurationManager);
var msg =
string.Format(CultureInfo.InvariantCulture, "map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}, maxRetry {4}, allowedFailedEvaluators {5}.",
memoryPerMapper,
memoryForUpdateTask,
coresPerMapper,
coresForUpdateTask,
_maxRetryNumberForFaultTolerant,
allowedFailedEvaluators);
Logger.Log(Level.Info, msg);
}
/// <summary>
/// Create an EvaluatorManager for testing
/// </summary>
/// <param name="totalEvaluators"></param>
/// <param name="allowedNumberOfEvaluatorFailures"></param>
/// <returns></returns>
private EvaluatorManager CreateEvaluatorManager(int totalEvaluators, int allowedNumberOfEvaluatorFailures)
{
var updateSpec = new EvaluatorSpecification(500, 2);
var mapperSpec = new EvaluatorSpecification(1000, 3);
return new EvaluatorManager(totalEvaluators, allowedNumberOfEvaluatorFailures, CreateMockEvaluatorRequestor(), updateSpec, mapperSpec);
}