private ContextAndServiceConfiguration GetDataLoadingContextAndServiceConfiguration( IPartitionDescriptor partitionDescriptor, string evaluatorId) { var dataLoadingContextConf = TangFactory.GetTang() .NewConfigurationBuilder() .BindSetEntry <ContextConfigurationOptions.StartHandlers, DataLoadingContext <TPartitionType>, IObserver <IContextStart> >( GenericType <ContextConfigurationOptions.StartHandlers> .Class, GenericType <DataLoadingContext <TPartitionType> > .Class) .Build(); var serviceConf = TangFactory.GetTang() .NewConfigurationBuilder(ServiceConfiguration.ConfigurationModule.Build(), dataLoadingContextConf, partitionDescriptor.GetPartitionConfiguration()) .Build(); var contextConf = ContextConfiguration.ConfigurationModule .Set(ContextConfiguration.Identifier, string.Format("DataLoading-{0}", evaluatorId)) .Build(); return(new ContextAndServiceConfiguration(contextConf, serviceConf)); }
/// <summary> /// Gives context and service configuration for next evaluator either from failed /// evaluator or new configuration /// </summary> /// <param name="evaluatorId"></param> /// <returns></returns> internal ContextAndServiceConfiguration GetDataLoadingConfigurationForEvaluatorById(string evaluatorId) { if (_partitionDescriptorIds.Count == 0) { Exceptions.Throw(new IMRUSystemException("No more data configuration can be provided"), Logger); } if (_partitionIdProvider.ContainsKey(evaluatorId)) { var msg = string.Format( CultureInfo.InvariantCulture, "Evaluator Id:{0} already present in configuration cache, they have to be unique", evaluatorId); Exceptions.Throw(new IMRUSystemException(msg), Logger); } Logger.Log(Level.Info, "Getting a new data loading configuration"); _partitionIdProvider[evaluatorId] = _partitionDescriptorIds.Pop(); try { IPartitionDescriptor partitionDescriptor = _dataset.GetPartitionDescriptorForId(_partitionIdProvider[evaluatorId]); return(GetDataLoadingContextAndServiceConfiguration(partitionDescriptor, evaluatorId)); } catch (Exception e) { var msg = string.Format(CultureInfo.InvariantCulture, "Error while trying to access partition descriptor:{0} from dataset", _partitionIdProvider[evaluatorId]); Exceptions.Throw(e, msg, Logger); return(null); } }
/// <summary> /// Creates service and data loading context configuration for given context id and partition descriptor /// </summary> /// <param name="partitionDescriptor"></param> /// <param name="contextId"></param> /// <returns>Configuration for context and service</returns> private ContextAndServiceConfiguration GetDataLoadingContextAndServiceConfiguration( IPartitionDescriptor partitionDescriptor, string contextId) { var dataLoadingContextConf = TangFactory.GetTang() .NewConfigurationBuilder() .BindSetEntry <ContextConfigurationOptions.StartHandlers, DataLoadingContext <TPartitionType>, IObserver <IContextStart> >( GenericType <ContextConfigurationOptions.StartHandlers> .Class, GenericType <DataLoadingContext <TPartitionType> > .Class) .Build(); var serviceConf = TangFactory.GetTang() .NewConfigurationBuilder( ServiceConfiguration.ConfigurationModule .Set(ServiceConfiguration.Services, GenericType <TaskStateService> .Class) .Build(), dataLoadingContextConf, partitionDescriptor.GetPartitionConfiguration(), _configurationManager.MapTaskStateConfiguration) .Build(); var contextConf = ContextConfiguration.ConfigurationModule .Set(ContextConfiguration.Identifier, contextId) .Build(); return(new ContextAndServiceConfiguration(contextConf, serviceConf)); }
/// <summary> /// Gives context and service configuration for next evaluator either from failed /// evaluator or new configuration /// </summary> /// <param name="evaluatorId"></param> /// <returns></returns> private ContextAndServiceConfiguration GetDataLoadingConfigurationForEvaluatorById(string evaluatorId) { string msg; if (_contextLoadedEvaluators.Contains(evaluatorId)) { msg = string.Format("Evaluator:{0} already has the data loaded", evaluatorId); Exceptions.Throw(new IMRUSystemException(msg), Logger); } if (_partitionDescriptorIds.Count == 0) { Exceptions.Throw(new IMRUSystemException("No more data configuration can be provided"), Logger); } if (_partitionIdProvider.ContainsKey(evaluatorId)) { msg = string.Format( "Evaluator Id:{0} already present in configuration cache, they have to be unique", evaluatorId); Exceptions.Throw(new IMRUSystemException(msg), Logger); } Logger.Log(Level.Info, "Getting a new data loading configuration"); _partitionIdProvider[evaluatorId] = _partitionDescriptorIds.Pop(); _submittedEvaluators.Add(evaluatorId); msg = string.Format( "Current status: Submitted Evaluators-{0}, Data Loaded Evaluators-{1}, Unused data partitions-{2}", _submittedEvaluators.Count, _contextLoadedEvaluators.Count, _partitionDescriptorIds.Count); Logger.Log(Level.Info, msg); try { IPartitionDescriptor partitionDescriptor = _dataset.GetPartitionDescriptorForId(_partitionIdProvider[evaluatorId]); return(GetDataLoadingContextAndServiceConfiguration(partitionDescriptor, evaluatorId)); } catch (Exception e) { msg = string.Format("Error while trying to access partition descriptor:{0} from dataset", _partitionIdProvider[evaluatorId]); Exceptions.Throw(e, msg, Logger); return(null); } }
/// <summary> /// Gives context and service configuration for next evaluator either from failed /// evaluator or new configuration /// </summary> /// <param name="evaluatorId"></param> /// <returns>Configuration for context and service</returns> internal ContextAndServiceConfiguration GetDataLoadingConfigurationForEvaluatorById(string evaluatorId) { try { Logger.Log(Level.Info, "Getting a new data loading configuration"); _partitionContextIdProvider.Add(evaluatorId, _availablePartitionDescriptorContextIds.Pop()); } catch (InvalidOperationException e) { throw new IMRUSystemException("No more data configuration can be provided", e); } catch (ArgumentException e) { var msg = string.Format( CultureInfo.InvariantCulture, "Evaluator Id:{0} already present in configuration cache, they have to be unique", evaluatorId); throw new IMRUSystemException(msg, e); } try { var partitionIdContextId = _partitionContextIdProvider[evaluatorId]; IPartitionDescriptor partitionDescriptor = _dataset.GetPartitionDescriptorForId(partitionIdContextId.PartitionDescriptorId); return(GetDataLoadingContextAndServiceConfiguration(partitionDescriptor, partitionIdContextId.ContextId)); } catch (Exception e) { var msg = string.Format(CultureInfo.InvariantCulture, "Error while trying to access partition descriptor:{0} from dataset", _partitionContextIdProvider[evaluatorId]); Exceptions.Throw(e, msg, Logger); return(null); } }
private ContextAndServiceConfiguration GetMapTaskContextAndServiceConfiguration(IPartitionDescriptor partitionDescriptor) { var codecConfig = TangFactory.GetTang() .NewConfigurationBuilder( StreamingCodecConfiguration <MapInputWithControlMessage <TMapInput> > .Conf.Set( StreamingCodecConfiguration <MapInputWithControlMessage <TMapInput> > .Codec, GenericType <MapInputWithControlMessageCodec <TMapInput> > .Class).Build(), StreamingCodecConfigurationMinusMessage <TMapOutput> .Conf.Build(), _configurationManager.MapInputCodecConfiguration) .Build(); var contextConf = _groupCommDriver.GetContextConfiguration(); var serviceConf = Configurations.Merge(_groupCommDriver.GetServiceConfiguration(), codecConfig, partitionDescriptor.GetPartitionConfiguration()); return(new ContextAndServiceConfiguration(contextConf, serviceConf)); }