private ContextAndServiceConfiguration GetDataLoadingContextAndServiceConfiguration(
IPartitionDescriptor partitionDescriptor,
string evaluatorId)
{
var dataLoadingContextConf =
TangFactory.GetTang()
.NewConfigurationBuilder()
.BindSetEntry <ContextConfigurationOptions.StartHandlers, DataLoadingContext <TPartitionType>, IObserver <IContextStart> >(
GenericType <ContextConfigurationOptions.StartHandlers> .Class,
GenericType <DataLoadingContext <TPartitionType> > .Class)
.Build();
var serviceConf =
TangFactory.GetTang()
.NewConfigurationBuilder(ServiceConfiguration.ConfigurationModule.Build(),
dataLoadingContextConf,
partitionDescriptor.GetPartitionConfiguration())
.Build();
var contextConf = ContextConfiguration.ConfigurationModule
.Set(ContextConfiguration.Identifier, string.Format("DataLoading-{0}", evaluatorId))
.Build();
return(new ContextAndServiceConfiguration(contextConf, serviceConf));
}
/// <summary>
/// Gives context and service configuration for next evaluator either from failed
/// evaluator or new configuration
/// </summary>
/// <param name="evaluatorId"></param>
/// <returns></returns>
internal ContextAndServiceConfiguration GetDataLoadingConfigurationForEvaluatorById(string evaluatorId)
{
if (_partitionDescriptorIds.Count == 0)
{
Exceptions.Throw(new IMRUSystemException("No more data configuration can be provided"), Logger);
}
if (_partitionIdProvider.ContainsKey(evaluatorId))
{
var msg =
string.Format(
CultureInfo.InvariantCulture,
"Evaluator Id:{0} already present in configuration cache, they have to be unique",
evaluatorId);
Exceptions.Throw(new IMRUSystemException(msg), Logger);
}
Logger.Log(Level.Info, "Getting a new data loading configuration");
_partitionIdProvider[evaluatorId] = _partitionDescriptorIds.Pop();
try
{
IPartitionDescriptor partitionDescriptor =
_dataset.GetPartitionDescriptorForId(_partitionIdProvider[evaluatorId]);
return(GetDataLoadingContextAndServiceConfiguration(partitionDescriptor, evaluatorId));
}
catch (Exception e)
{
var msg = string.Format(CultureInfo.InvariantCulture, "Error while trying to access partition descriptor:{0} from dataset",
_partitionIdProvider[evaluatorId]);
Exceptions.Throw(e, msg, Logger);
return(null);
}
}
/// <summary>
/// Creates service and data loading context configuration for given context id and partition descriptor
/// </summary>
/// <param name="partitionDescriptor"></param>
/// <param name="contextId"></param>
/// <returns>Configuration for context and service</returns>
private ContextAndServiceConfiguration GetDataLoadingContextAndServiceConfiguration(
IPartitionDescriptor partitionDescriptor, string contextId)
{
var dataLoadingContextConf =
TangFactory.GetTang()
.NewConfigurationBuilder()
.BindSetEntry <ContextConfigurationOptions.StartHandlers, DataLoadingContext <TPartitionType>, IObserver <IContextStart> >(
GenericType <ContextConfigurationOptions.StartHandlers> .Class,
GenericType <DataLoadingContext <TPartitionType> > .Class)
.Build();
var serviceConf =
TangFactory.GetTang()
.NewConfigurationBuilder(
ServiceConfiguration.ConfigurationModule
.Set(ServiceConfiguration.Services, GenericType <TaskStateService> .Class)
.Build(),
dataLoadingContextConf,
partitionDescriptor.GetPartitionConfiguration(),
_configurationManager.MapTaskStateConfiguration)
.Build();
var contextConf = ContextConfiguration.ConfigurationModule
.Set(ContextConfiguration.Identifier, contextId)
.Build();
return(new ContextAndServiceConfiguration(contextConf, serviceConf));
}
/// <summary>
/// Gives context and service configuration for next evaluator either from failed
/// evaluator or new configuration
/// </summary>
/// <param name="evaluatorId"></param>
/// <returns></returns>
private ContextAndServiceConfiguration GetDataLoadingConfigurationForEvaluatorById(string evaluatorId)
{
string msg;
if (_contextLoadedEvaluators.Contains(evaluatorId))
{
msg = string.Format("Evaluator:{0} already has the data loaded", evaluatorId);
Exceptions.Throw(new IMRUSystemException(msg), Logger);
}
if (_partitionDescriptorIds.Count == 0)
{
Exceptions.Throw(new IMRUSystemException("No more data configuration can be provided"), Logger);
}
if (_partitionIdProvider.ContainsKey(evaluatorId))
{
msg =
string.Format(
"Evaluator Id:{0} already present in configuration cache, they have to be unique",
evaluatorId);
Exceptions.Throw(new IMRUSystemException(msg), Logger);
}
Logger.Log(Level.Info, "Getting a new data loading configuration");
_partitionIdProvider[evaluatorId] = _partitionDescriptorIds.Pop();
_submittedEvaluators.Add(evaluatorId);
msg = string.Format(
"Current status: Submitted Evaluators-{0}, Data Loaded Evaluators-{1}, Unused data partitions-{2}",
_submittedEvaluators.Count,
_contextLoadedEvaluators.Count,
_partitionDescriptorIds.Count);
Logger.Log(Level.Info, msg);
try
{
IPartitionDescriptor partitionDescriptor =
_dataset.GetPartitionDescriptorForId(_partitionIdProvider[evaluatorId]);
return(GetDataLoadingContextAndServiceConfiguration(partitionDescriptor, evaluatorId));
}
catch (Exception e)
{
msg = string.Format("Error while trying to access partition descriptor:{0} from dataset",
_partitionIdProvider[evaluatorId]);
Exceptions.Throw(e, msg, Logger);
return(null);
}
}
/// <summary>
/// Gives context and service configuration for next evaluator either from failed
/// evaluator or new configuration
/// </summary>
/// <param name="evaluatorId"></param>
/// <returns>Configuration for context and service</returns>
internal ContextAndServiceConfiguration GetDataLoadingConfigurationForEvaluatorById(string evaluatorId)
{
try
{
Logger.Log(Level.Info, "Getting a new data loading configuration");
_partitionContextIdProvider.Add(evaluatorId, _availablePartitionDescriptorContextIds.Pop());
}
catch (InvalidOperationException e)
{
throw new IMRUSystemException("No more data configuration can be provided", e);
}
catch (ArgumentException e)
{
var msg =
string.Format(
CultureInfo.InvariantCulture,
"Evaluator Id:{0} already present in configuration cache, they have to be unique",
evaluatorId);
throw new IMRUSystemException(msg, e);
}
try
{
var partitionIdContextId = _partitionContextIdProvider[evaluatorId];
IPartitionDescriptor partitionDescriptor =
_dataset.GetPartitionDescriptorForId(partitionIdContextId.PartitionDescriptorId);
return(GetDataLoadingContextAndServiceConfiguration(partitionDescriptor, partitionIdContextId.ContextId));
}
catch (Exception e)
{
var msg = string.Format(CultureInfo.InvariantCulture,
"Error while trying to access partition descriptor:{0} from dataset",
_partitionContextIdProvider[evaluatorId]);
Exceptions.Throw(e, msg, Logger);
return(null);
}
}
private ContextAndServiceConfiguration GetMapTaskContextAndServiceConfiguration(IPartitionDescriptor partitionDescriptor)
{
var codecConfig =
TangFactory.GetTang()
.NewConfigurationBuilder(
StreamingCodecConfiguration <MapInputWithControlMessage <TMapInput> > .Conf.Set(
StreamingCodecConfiguration <MapInputWithControlMessage <TMapInput> > .Codec,
GenericType <MapInputWithControlMessageCodec <TMapInput> > .Class).Build(),
StreamingCodecConfigurationMinusMessage <TMapOutput> .Conf.Build(),
_configurationManager.MapInputCodecConfiguration)
.Build();
var contextConf = _groupCommDriver.GetContextConfiguration();
var serviceConf = Configurations.Merge(_groupCommDriver.GetServiceConfiguration(), codecConfig, partitionDescriptor.GetPartitionConfiguration());
return(new ContextAndServiceConfiguration(contextConf, serviceConf));
}
