public PipelinedBroadcastReduceDriver(
[Parameter(typeof(GroupTestConfig.NumEvaluators))] int numEvaluators,
[Parameter(typeof(GroupTestConfig.NumIterations))] int numIterations,
[Parameter(typeof(GroupTestConfig.StartingPort))] int startingPort,
[Parameter(typeof(GroupTestConfig.PortRange))] int portRange,
[Parameter(typeof(GroupTestConfig.ChunkSize))] int chunkSize,
[Parameter(typeof(GroupTestConfig.ArraySize))] int arraySize,
GroupCommDriver groupCommDriver,
IEvaluatorRequestor evaluatorRequestor)
{
Logger.Log(Level.Info, "entering the driver code " + chunkSize);
_numEvaluators = numEvaluators;
_numIterations = numIterations;
_arraySize = arraySize;
_evaluatorRequestor = evaluatorRequestor;
_tcpPortProviderConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindNamedParameter <TcpPortRangeStart, int>(GenericType <TcpPortRangeStart> .Class,
startingPort.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter <TcpPortRangeCount, int>(GenericType <TcpPortRangeCount> .Class,
portRange.ToString(CultureInfo.InvariantCulture))
.Build();
_codecConfig = StreamingCodecConfiguration <int[]> .Conf
.Set(StreamingCodecConfiguration <int[]> .Codec, GenericType <IntArrayStreamingCodec> .Class)
.Build();
var reduceFunctionConfig = ReduceFunctionConfiguration <int[]> .Conf
.Set(ReduceFunctionConfiguration <int[]> .ReduceFunction, GenericType <ArraySumFunction> .Class)
.Build();
var dataConverterConfig = TangFactory.GetTang().NewConfigurationBuilder(
PipelineDataConverterConfiguration <int[]> .Conf
.Set(PipelineDataConverterConfiguration <int[]> .DataConverter,
GenericType <PipelineIntDataConverter> .Class)
.Build())
.BindNamedParameter <GroupTestConfig.ChunkSize, int>(
GenericType <GroupTestConfig.ChunkSize> .Class,
chunkSize.ToString(CultureInfo.InvariantCulture))
.Build();
_groupCommDriver = groupCommDriver;
_commGroup = _groupCommDriver.DefaultGroup
.AddBroadcast <int[]>(
GroupTestConstants.BroadcastOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
dataConverterConfig)
.AddReduce <int[]>(
GroupTestConstants.ReduceOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
reduceFunctionConfig,
dataConverterConfig)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, numEvaluators);
}
private KMeansDriverHandlers(
[Parameter(typeof(NumPartitions))] int numPartitions,
GroupCommDriver groupCommDriver,
IEvaluatorRequestor evaluatorRequestor,
CommandLineArguments arguments)
{
_executionDirectory = Path.Combine(Directory.GetCurrentDirectory(), Constants.KMeansExecutionBaseDirectory, Guid.NewGuid().ToString("N").Substring(0, 4));
string dataFile = arguments.Arguments.First();
DataVector.ShuffleDataAndGetInitialCentriods(
dataFile,
numPartitions,
_clustersNumber,
_executionDirectory);
_totalEvaluators = numPartitions + 1;
_groupCommDriver = groupCommDriver;
_evaluatorRequestor = evaluatorRequestor;
_centroidCodecConf = CodecToStreamingCodecConfiguration <Centroids> .Conf
.Set(CodecToStreamingCodecConfiguration <Centroids> .Codec, GenericType <CentroidsCodec> .Class)
.Build();
IConfiguration dataConverterConfig1 = PipelineDataConverterConfiguration <Centroids> .Conf
.Set(PipelineDataConverterConfiguration <Centroids> .DataConverter, GenericType <DefaultPipelineDataConverter <Centroids> > .Class)
.Build();
_controlMessageCodecConf = CodecToStreamingCodecConfiguration <ControlMessage> .Conf
.Set(CodecToStreamingCodecConfiguration <ControlMessage> .Codec, GenericType <ControlMessageCodec> .Class)
.Build();
IConfiguration dataConverterConfig2 = PipelineDataConverterConfiguration <ControlMessage> .Conf
.Set(PipelineDataConverterConfiguration <ControlMessage> .DataConverter, GenericType <DefaultPipelineDataConverter <ControlMessage> > .Class)
.Build();
_processedResultsCodecConf = CodecToStreamingCodecConfiguration <ProcessedResults> .Conf
.Set(CodecToStreamingCodecConfiguration <ProcessedResults> .Codec, GenericType <ProcessedResultsCodec> .Class)
.Build();
IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <ProcessedResults> .Conf
.Set(ReduceFunctionConfiguration <ProcessedResults> .ReduceFunction, GenericType <KMeansMasterTask.AggregateMeans> .Class)
.Build();
IConfiguration dataConverterConfig3 = PipelineDataConverterConfiguration <ProcessedResults> .Conf
.Set(PipelineDataConverterConfiguration <ProcessedResults> .DataConverter, GenericType <DefaultPipelineDataConverter <ProcessedResults> > .Class)
.Build();
_commGroup = _groupCommDriver.DefaultGroup
.AddBroadcast <Centroids>(Constants.CentroidsBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig1)
.AddBroadcast <ControlMessage>(Constants.ControlMessageBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig2)
.AddReduce <ProcessedResults>(Constants.MeansReduceOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, reduceFunctionConfig, dataConverterConfig3)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, _totalEvaluators);
}
public KMeansDriverHandlers([Parameter(typeof(NumPartitions))] int numPartitions, GroupCommDriver groupCommDriver)
{
Identifier = "KMeansDriverId";
_executionDirectory = Path.Combine(Directory.GetCurrentDirectory(), Constants.KMeansExecutionBaseDirectory, Guid.NewGuid().ToString("N").Substring(0, 4));
ISet <string> arguments = ClrHandlerHelper.GetCommandLineArguments();
string dataFile = arguments.Single(a => a.StartsWith("DataFile", StringComparison.Ordinal)).Split(':')[1];
DataVector.ShuffleDataAndGetInitialCentriods(
Path.Combine(Directory.GetCurrentDirectory(), "reef", "global", dataFile),
numPartitions,
_clustersNumber,
_executionDirectory);
_totalEvaluators = numPartitions + 1;
_groupCommDriver = groupCommDriver;
_centroidCodecConf = CodecToStreamingCodecConfiguration <Centroids> .Conf
.Set(CodecConfiguration <Centroids> .Codec, GenericType <CentroidsCodec> .Class)
.Build();
IConfiguration dataConverterConfig1 = PipelineDataConverterConfiguration <Centroids> .Conf
.Set(PipelineDataConverterConfiguration <Centroids> .DataConverter, GenericType <DefaultPipelineDataConverter <Centroids> > .Class)
.Build();
_controlMessageCodecConf = CodecToStreamingCodecConfiguration <ControlMessage> .Conf
.Set(CodecConfiguration <ControlMessage> .Codec, GenericType <ControlMessageCodec> .Class)
.Build();
IConfiguration dataConverterConfig2 = PipelineDataConverterConfiguration <ControlMessage> .Conf
.Set(PipelineDataConverterConfiguration <ControlMessage> .DataConverter, GenericType <DefaultPipelineDataConverter <ControlMessage> > .Class)
.Build();
_processedResultsCodecConf = CodecToStreamingCodecConfiguration <ProcessedResults> .Conf
.Set(CodecConfiguration <ProcessedResults> .Codec, GenericType <ProcessedResultsCodec> .Class)
.Build();
IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <ProcessedResults> .Conf
.Set(ReduceFunctionConfiguration <ProcessedResults> .ReduceFunction, GenericType <KMeansMasterTask.AggregateMeans> .Class)
.Build();
IConfiguration dataConverterConfig3 = PipelineDataConverterConfiguration <ProcessedResults> .Conf
.Set(PipelineDataConverterConfiguration <ProcessedResults> .DataConverter, GenericType <DefaultPipelineDataConverter <ProcessedResults> > .Class)
.Build();
_commGroup = _groupCommDriver.DefaultGroup
.AddBroadcast <Centroids>(Constants.CentroidsBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig1)
.AddBroadcast <ControlMessage>(Constants.ControlMessageBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig2)
.AddReduce <ProcessedResults>(Constants.MeansReduceOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, reduceFunctionConfig, dataConverterConfig3)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, _totalEvaluators);
CreateClassHierarchy();
}
public BroadcastReduceDriver(
[Parameter(typeof(GroupTestConfig.NumEvaluators))] int numEvaluators,
[Parameter(typeof(GroupTestConfig.NumIterations))] int numIterations,
[Parameter(typeof(GroupTestConfig.StartingPort))] int startingPort,
[Parameter(typeof(GroupTestConfig.PortRange))] int portRange,
GroupCommDriver groupCommDriver)
{
Identifier = "BroadcastStartHandler";
_numEvaluators = numEvaluators;
_numIterations = numIterations;
_groupCommDriver = groupCommDriver;
_tcpPortProviderConfig = TangFactory.GetTang().NewConfigurationBuilder()
.BindNamedParameter <TcpPortRangeStart, int>(GenericType <TcpPortRangeStart> .Class,
startingPort.ToString(CultureInfo.InvariantCulture))
.BindNamedParameter <TcpPortRangeCount, int>(GenericType <TcpPortRangeCount> .Class,
portRange.ToString(CultureInfo.InvariantCulture))
.Build();
_codecConfig = StreamingCodecConfiguration <int> .Conf
.Set(StreamingCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class)
.Build();
IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <int> .Conf
.Set(ReduceFunctionConfiguration <int> .ReduceFunction, GenericType <SumFunction> .Class)
.Build();
IConfiguration dataConverterConfig = PipelineDataConverterConfiguration <int> .Conf
.Set(PipelineDataConverterConfiguration <int> .DataConverter, GenericType <DefaultPipelineDataConverter <int> > .Class)
.Build();
_commGroup = _groupCommDriver.DefaultGroup
.AddBroadcast <int>(
GroupTestConstants.BroadcastOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
dataConverterConfig)
.AddReduce <int>(
GroupTestConstants.ReduceOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
reduceFunctionConfig,
dataConverterConfig)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, numEvaluators);
CreateClassHierarchy();
}
private IMRUDriver(IPartitionedInputDataSet dataSet,
[Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs,
ConfigurationManager configurationManager,
IEvaluatorRequestor evaluatorRequestor,
[Parameter(typeof(CoresPerMapper))] int coresPerMapper,
[Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask,
[Parameter(typeof(MemoryPerMapper))] int memoryPerMapper,
[Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask,
[Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction,
[Parameter(typeof(InvokeGC))] bool invokeGC,
IGroupCommDriver groupCommDriver)
{
_dataSet = dataSet;
_configurationManager = configurationManager;
_evaluatorRequestor = evaluatorRequestor;
_groupCommDriver = groupCommDriver;
_coresPerMapper = coresPerMapper;
_coresForUpdateTask = coresForUpdateTask;
_memoryPerMapper = memoryPerMapper;
_memoryForUpdateTask = memoryForUpdateTask;
_perMapperConfigs = perMapperConfigs;
_completedTasks = new ConcurrentBag <ICompletedTask>();
_allowedFailedEvaluators = (int)(failedEvaluatorsFraction * dataSet.Count);
_invokeGC = invokeGC;
AddGroupCommunicationOperators();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, _dataSet.Count + 1);
_taskIdStack = new ConcurrentStack <string>();
_perMapperConfiguration = new ConcurrentStack <IConfiguration>();
_partitionDescriptorStack = new Stack <IPartitionDescriptor>();
ConstructTaskIdAndPartitionDescriptorStack();
_serviceAndContextConfigurationProvider =
new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput>(dataSet.Count + 1, groupCommDriver,
_configurationManager, _partitionDescriptorStack);
var msg = string.Format("map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}",
_memoryPerMapper, _memoryForUpdateTask, _coresPerMapper, _coresForUpdateTask);
Logger.Log(Level.Info, msg);
}
public ScatterReduceDriver(
[Parameter(typeof(GroupTestConfig.NumEvaluators))] int numEvaluators,
GroupCommDriver groupCommDriver)
{
Identifier = "BroadcastStartHandler";
_numEvaluators = numEvaluators;
_groupCommDriver = groupCommDriver;
_codecConfig = StreamingCodecConfiguration <int> .Conf
.Set(StreamingCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class)
.Build();
IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <int> .Conf
.Set(ReduceFunctionConfiguration <int> .ReduceFunction, GenericType <SumFunction> .Class)
.Build();
IConfiguration dataConverterConfig = PipelineDataConverterConfiguration <int> .Conf
.Set(PipelineDataConverterConfiguration <int> .DataConverter, GenericType <DefaultPipelineDataConverter <int> > .Class)
.Build();
_commGroup = _groupCommDriver.DefaultGroup
.AddScatter <int>(
GroupTestConstants.ScatterOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
dataConverterConfig)
.AddReduce <int>(
GroupTestConstants.ReduceOperatorName,
GroupTestConstants.MasterTaskId,
TopologyTypes.Tree,
reduceFunctionConfig,
dataConverterConfig)
.Build();
_groupCommTaskStarter = new TaskStarter(_groupCommDriver, numEvaluators);
CreateClassHierarchy();
}
public void Activity(string name, int duration, string[] resources, int[] count, Color color, int maxPlanDuration, TaskStarter func)
{
t.Activity(name, duration, resources, count, color, maxPlanDuration, func);
}
/// <summary>
/// Executes an <see cref="Func{ICatalogsContainer, TReturn}"/> that takes an <see cref="ICatalogsContainer"/> as parameter within
/// a using block.
/// </summary>
/// <typeparam name="TReturn">The <see cref="Type"/> of the instance that is returned.</typeparam>
/// <param name="function">The <see cref="Func{ICatalogsContainer, TReturn}"/> that is executed within the using block.</param>
/// <returns>An instance of the specified <see cref="Type"/>.</returns>
public static async Task <TReturn> ExecuteWithCatalogScopeAsync <TReturn>(Func <ICatalogsContainer, TReturn> function)
{
return(await TaskStarter.StartNew(() => ExecuteWithCatalogScope(function)));
}
/// <summary>
/// Executes an <see cref="Action{ICatalogsContainer}"/> that takes an <see cref="ICatalogsContainer"/> as parameter within
/// a using block.
/// </summary>
/// <param name="action">The <see cref="Action{ICatalogsContainer}"/> that is executed within the using block.</param>
public static async Task ExecuteWithCatalogScopeAsync(Action <ICatalogsContainer> action)
{
await TaskStarter.StartNew(() => ExecuteWithCatalogScope(action));
}
public AllServices()
{
restApp = new RestApp();
taskStarter = new TaskStarter();
}
public void Activity(string name, int duration, string[] resources, int[] count, Color color, int maxPlanDuration, TaskStarter func)
{
if (ScheduleMode)
{
ScheduleActivity act = new ScheduleActivity();
act.Name = name;
act.Duration = duration;
act.MaxPlannedDuration = maxPlanDuration;
act.Color = color;
for (int i = 0; resources != null && i < resources.Length; i++)
{
act.Resources[resources[i]] = count[i];
}
embedActivities.Add(act);
}
else
{
if (Status == Status.Cancelled)
{
return;
}
if (CurrentActivity == null)
{
CurrentActivity = Activities[0];
}
else
{
CurrentActivity = CurrentActivity.Next;
}
WaitForActivityStart(CurrentActivity);
func();
}
}
