public PipelinedBroadcastReduceDriver( [Parameter(typeof(GroupTestConfig.NumEvaluators))] int numEvaluators, [Parameter(typeof(GroupTestConfig.NumIterations))] int numIterations, [Parameter(typeof(GroupTestConfig.StartingPort))] int startingPort, [Parameter(typeof(GroupTestConfig.PortRange))] int portRange, [Parameter(typeof(GroupTestConfig.ChunkSize))] int chunkSize, [Parameter(typeof(GroupTestConfig.ArraySize))] int arraySize, GroupCommDriver groupCommDriver, IEvaluatorRequestor evaluatorRequestor) { Logger.Log(Level.Info, "entering the driver code " + chunkSize); _numEvaluators = numEvaluators; _numIterations = numIterations; _arraySize = arraySize; _evaluatorRequestor = evaluatorRequestor; _tcpPortProviderConfig = TangFactory.GetTang().NewConfigurationBuilder() .BindNamedParameter <TcpPortRangeStart, int>(GenericType <TcpPortRangeStart> .Class, startingPort.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter <TcpPortRangeCount, int>(GenericType <TcpPortRangeCount> .Class, portRange.ToString(CultureInfo.InvariantCulture)) .Build(); _codecConfig = StreamingCodecConfiguration <int[]> .Conf .Set(StreamingCodecConfiguration <int[]> .Codec, GenericType <IntArrayStreamingCodec> .Class) .Build(); var reduceFunctionConfig = ReduceFunctionConfiguration <int[]> .Conf .Set(ReduceFunctionConfiguration <int[]> .ReduceFunction, GenericType <ArraySumFunction> .Class) .Build(); var dataConverterConfig = TangFactory.GetTang().NewConfigurationBuilder( PipelineDataConverterConfiguration <int[]> .Conf .Set(PipelineDataConverterConfiguration <int[]> .DataConverter, GenericType <PipelineIntDataConverter> .Class) .Build()) .BindNamedParameter <GroupTestConfig.ChunkSize, int>( GenericType <GroupTestConfig.ChunkSize> .Class, chunkSize.ToString(CultureInfo.InvariantCulture)) .Build(); _groupCommDriver = groupCommDriver; _commGroup = _groupCommDriver.DefaultGroup .AddBroadcast <int[]>( GroupTestConstants.BroadcastOperatorName, GroupTestConstants.MasterTaskId, TopologyTypes.Tree, dataConverterConfig) .AddReduce <int[]>( GroupTestConstants.ReduceOperatorName, GroupTestConstants.MasterTaskId, TopologyTypes.Tree, reduceFunctionConfig, dataConverterConfig) .Build(); _groupCommTaskStarter = new TaskStarter(_groupCommDriver, numEvaluators); }
private KMeansDriverHandlers( [Parameter(typeof(NumPartitions))] int numPartitions, GroupCommDriver groupCommDriver, IEvaluatorRequestor evaluatorRequestor, CommandLineArguments arguments) { _executionDirectory = Path.Combine(Directory.GetCurrentDirectory(), Constants.KMeansExecutionBaseDirectory, Guid.NewGuid().ToString("N").Substring(0, 4)); string dataFile = arguments.Arguments.First(); DataVector.ShuffleDataAndGetInitialCentriods( dataFile, numPartitions, _clustersNumber, _executionDirectory); _totalEvaluators = numPartitions + 1; _groupCommDriver = groupCommDriver; _evaluatorRequestor = evaluatorRequestor; _centroidCodecConf = CodecToStreamingCodecConfiguration <Centroids> .Conf .Set(CodecToStreamingCodecConfiguration <Centroids> .Codec, GenericType <CentroidsCodec> .Class) .Build(); IConfiguration dataConverterConfig1 = PipelineDataConverterConfiguration <Centroids> .Conf .Set(PipelineDataConverterConfiguration <Centroids> .DataConverter, GenericType <DefaultPipelineDataConverter <Centroids> > .Class) .Build(); _controlMessageCodecConf = CodecToStreamingCodecConfiguration <ControlMessage> .Conf .Set(CodecToStreamingCodecConfiguration <ControlMessage> .Codec, GenericType <ControlMessageCodec> .Class) .Build(); IConfiguration dataConverterConfig2 = PipelineDataConverterConfiguration <ControlMessage> .Conf .Set(PipelineDataConverterConfiguration <ControlMessage> .DataConverter, GenericType <DefaultPipelineDataConverter <ControlMessage> > .Class) .Build(); _processedResultsCodecConf = CodecToStreamingCodecConfiguration <ProcessedResults> .Conf .Set(CodecToStreamingCodecConfiguration <ProcessedResults> .Codec, GenericType <ProcessedResultsCodec> .Class) .Build(); IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <ProcessedResults> .Conf .Set(ReduceFunctionConfiguration <ProcessedResults> .ReduceFunction, GenericType <KMeansMasterTask.AggregateMeans> .Class) .Build(); IConfiguration dataConverterConfig3 = PipelineDataConverterConfiguration <ProcessedResults> .Conf .Set(PipelineDataConverterConfiguration <ProcessedResults> .DataConverter, GenericType <DefaultPipelineDataConverter <ProcessedResults> > .Class) .Build(); _commGroup = _groupCommDriver.DefaultGroup .AddBroadcast <Centroids>(Constants.CentroidsBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig1) .AddBroadcast <ControlMessage>(Constants.ControlMessageBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig2) .AddReduce <ProcessedResults>(Constants.MeansReduceOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, reduceFunctionConfig, dataConverterConfig3) .Build(); _groupCommTaskStarter = new TaskStarter(_groupCommDriver, _totalEvaluators); }
public KMeansDriverHandlers([Parameter(typeof(NumPartitions))] int numPartitions, GroupCommDriver groupCommDriver) { Identifier = "KMeansDriverId"; _executionDirectory = Path.Combine(Directory.GetCurrentDirectory(), Constants.KMeansExecutionBaseDirectory, Guid.NewGuid().ToString("N").Substring(0, 4)); ISet <string> arguments = ClrHandlerHelper.GetCommandLineArguments(); string dataFile = arguments.Single(a => a.StartsWith("DataFile", StringComparison.Ordinal)).Split(':')[1]; DataVector.ShuffleDataAndGetInitialCentriods( Path.Combine(Directory.GetCurrentDirectory(), "reef", "global", dataFile), numPartitions, _clustersNumber, _executionDirectory); _totalEvaluators = numPartitions + 1; _groupCommDriver = groupCommDriver; _centroidCodecConf = CodecToStreamingCodecConfiguration <Centroids> .Conf .Set(CodecConfiguration <Centroids> .Codec, GenericType <CentroidsCodec> .Class) .Build(); IConfiguration dataConverterConfig1 = PipelineDataConverterConfiguration <Centroids> .Conf .Set(PipelineDataConverterConfiguration <Centroids> .DataConverter, GenericType <DefaultPipelineDataConverter <Centroids> > .Class) .Build(); _controlMessageCodecConf = CodecToStreamingCodecConfiguration <ControlMessage> .Conf .Set(CodecConfiguration <ControlMessage> .Codec, GenericType <ControlMessageCodec> .Class) .Build(); IConfiguration dataConverterConfig2 = PipelineDataConverterConfiguration <ControlMessage> .Conf .Set(PipelineDataConverterConfiguration <ControlMessage> .DataConverter, GenericType <DefaultPipelineDataConverter <ControlMessage> > .Class) .Build(); _processedResultsCodecConf = CodecToStreamingCodecConfiguration <ProcessedResults> .Conf .Set(CodecConfiguration <ProcessedResults> .Codec, GenericType <ProcessedResultsCodec> .Class) .Build(); IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <ProcessedResults> .Conf .Set(ReduceFunctionConfiguration <ProcessedResults> .ReduceFunction, GenericType <KMeansMasterTask.AggregateMeans> .Class) .Build(); IConfiguration dataConverterConfig3 = PipelineDataConverterConfiguration <ProcessedResults> .Conf .Set(PipelineDataConverterConfiguration <ProcessedResults> .DataConverter, GenericType <DefaultPipelineDataConverter <ProcessedResults> > .Class) .Build(); _commGroup = _groupCommDriver.DefaultGroup .AddBroadcast <Centroids>(Constants.CentroidsBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig1) .AddBroadcast <ControlMessage>(Constants.ControlMessageBroadcastOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, dataConverterConfig2) .AddReduce <ProcessedResults>(Constants.MeansReduceOperatorName, Constants.MasterTaskId, TopologyTypes.Flat, reduceFunctionConfig, dataConverterConfig3) .Build(); _groupCommTaskStarter = new TaskStarter(_groupCommDriver, _totalEvaluators); CreateClassHierarchy(); }
public BroadcastReduceDriver( [Parameter(typeof(GroupTestConfig.NumEvaluators))] int numEvaluators, [Parameter(typeof(GroupTestConfig.NumIterations))] int numIterations, [Parameter(typeof(GroupTestConfig.StartingPort))] int startingPort, [Parameter(typeof(GroupTestConfig.PortRange))] int portRange, GroupCommDriver groupCommDriver) { Identifier = "BroadcastStartHandler"; _numEvaluators = numEvaluators; _numIterations = numIterations; _groupCommDriver = groupCommDriver; _tcpPortProviderConfig = TangFactory.GetTang().NewConfigurationBuilder() .BindNamedParameter <TcpPortRangeStart, int>(GenericType <TcpPortRangeStart> .Class, startingPort.ToString(CultureInfo.InvariantCulture)) .BindNamedParameter <TcpPortRangeCount, int>(GenericType <TcpPortRangeCount> .Class, portRange.ToString(CultureInfo.InvariantCulture)) .Build(); _codecConfig = StreamingCodecConfiguration <int> .Conf .Set(StreamingCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class) .Build(); IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <int> .Conf .Set(ReduceFunctionConfiguration <int> .ReduceFunction, GenericType <SumFunction> .Class) .Build(); IConfiguration dataConverterConfig = PipelineDataConverterConfiguration <int> .Conf .Set(PipelineDataConverterConfiguration <int> .DataConverter, GenericType <DefaultPipelineDataConverter <int> > .Class) .Build(); _commGroup = _groupCommDriver.DefaultGroup .AddBroadcast <int>( GroupTestConstants.BroadcastOperatorName, GroupTestConstants.MasterTaskId, TopologyTypes.Tree, dataConverterConfig) .AddReduce <int>( GroupTestConstants.ReduceOperatorName, GroupTestConstants.MasterTaskId, TopologyTypes.Tree, reduceFunctionConfig, dataConverterConfig) .Build(); _groupCommTaskStarter = new TaskStarter(_groupCommDriver, numEvaluators); CreateClassHierarchy(); }
private IMRUDriver(IPartitionedInputDataSet dataSet, [Parameter(typeof(PerMapConfigGeneratorSet))] ISet <IPerMapperConfigGenerator> perMapperConfigs, ConfigurationManager configurationManager, IEvaluatorRequestor evaluatorRequestor, [Parameter(typeof(CoresPerMapper))] int coresPerMapper, [Parameter(typeof(CoresForUpdateTask))] int coresForUpdateTask, [Parameter(typeof(MemoryPerMapper))] int memoryPerMapper, [Parameter(typeof(MemoryForUpdateTask))] int memoryForUpdateTask, [Parameter(typeof(AllowedFailedEvaluatorsFraction))] double failedEvaluatorsFraction, [Parameter(typeof(InvokeGC))] bool invokeGC, IGroupCommDriver groupCommDriver) { _dataSet = dataSet; _configurationManager = configurationManager; _evaluatorRequestor = evaluatorRequestor; _groupCommDriver = groupCommDriver; _coresPerMapper = coresPerMapper; _coresForUpdateTask = coresForUpdateTask; _memoryPerMapper = memoryPerMapper; _memoryForUpdateTask = memoryForUpdateTask; _perMapperConfigs = perMapperConfigs; _completedTasks = new ConcurrentBag <ICompletedTask>(); _allowedFailedEvaluators = (int)(failedEvaluatorsFraction * dataSet.Count); _invokeGC = invokeGC; AddGroupCommunicationOperators(); _groupCommTaskStarter = new TaskStarter(_groupCommDriver, _dataSet.Count + 1); _taskIdStack = new ConcurrentStack <string>(); _perMapperConfiguration = new ConcurrentStack <IConfiguration>(); _partitionDescriptorStack = new Stack <IPartitionDescriptor>(); ConstructTaskIdAndPartitionDescriptorStack(); _serviceAndContextConfigurationProvider = new ServiceAndContextConfigurationProvider <TMapInput, TMapOutput>(dataSet.Count + 1, groupCommDriver, _configurationManager, _partitionDescriptorStack); var msg = string.Format("map task memory:{0}, update task memory:{1}, map task cores:{2}, update task cores:{3}", _memoryPerMapper, _memoryForUpdateTask, _coresPerMapper, _coresForUpdateTask); Logger.Log(Level.Info, msg); }
public ScatterReduceDriver( [Parameter(typeof(GroupTestConfig.NumEvaluators))] int numEvaluators, GroupCommDriver groupCommDriver) { Identifier = "BroadcastStartHandler"; _numEvaluators = numEvaluators; _groupCommDriver = groupCommDriver; _codecConfig = StreamingCodecConfiguration <int> .Conf .Set(StreamingCodecConfiguration <int> .Codec, GenericType <IntStreamingCodec> .Class) .Build(); IConfiguration reduceFunctionConfig = ReduceFunctionConfiguration <int> .Conf .Set(ReduceFunctionConfiguration <int> .ReduceFunction, GenericType <SumFunction> .Class) .Build(); IConfiguration dataConverterConfig = PipelineDataConverterConfiguration <int> .Conf .Set(PipelineDataConverterConfiguration <int> .DataConverter, GenericType <DefaultPipelineDataConverter <int> > .Class) .Build(); _commGroup = _groupCommDriver.DefaultGroup .AddScatter <int>( GroupTestConstants.ScatterOperatorName, GroupTestConstants.MasterTaskId, TopologyTypes.Tree, dataConverterConfig) .AddReduce <int>( GroupTestConstants.ReduceOperatorName, GroupTestConstants.MasterTaskId, TopologyTypes.Tree, reduceFunctionConfig, dataConverterConfig) .Build(); _groupCommTaskStarter = new TaskStarter(_groupCommDriver, numEvaluators); CreateClassHierarchy(); }
public void Activity(string name, int duration, string[] resources, int[] count, Color color, int maxPlanDuration, TaskStarter func) { t.Activity(name, duration, resources, count, color, maxPlanDuration, func); }
/// <summary> /// Executes an <see cref="Func{ICatalogsContainer, TReturn}"/> that takes an <see cref="ICatalogsContainer"/> as parameter within /// a using block. /// </summary> /// <typeparam name="TReturn">The <see cref="Type"/> of the instance that is returned.</typeparam> /// <param name="function">The <see cref="Func{ICatalogsContainer, TReturn}"/> that is executed within the using block.</param> /// <returns>An instance of the specified <see cref="Type"/>.</returns> public static async Task <TReturn> ExecuteWithCatalogScopeAsync <TReturn>(Func <ICatalogsContainer, TReturn> function) { return(await TaskStarter.StartNew(() => ExecuteWithCatalogScope(function))); }
/// <summary> /// Executes an <see cref="Action{ICatalogsContainer}"/> that takes an <see cref="ICatalogsContainer"/> as parameter within /// a using block. /// </summary> /// <param name="action">The <see cref="Action{ICatalogsContainer}"/> that is executed within the using block.</param> public static async Task ExecuteWithCatalogScopeAsync(Action <ICatalogsContainer> action) { await TaskStarter.StartNew(() => ExecuteWithCatalogScope(action)); }
public AllServices() { restApp = new RestApp(); taskStarter = new TaskStarter(); }
public void Activity(string name, int duration, string[] resources, int[] count, Color color, int maxPlanDuration, TaskStarter func) { if (ScheduleMode) { ScheduleActivity act = new ScheduleActivity(); act.Name = name; act.Duration = duration; act.MaxPlannedDuration = maxPlanDuration; act.Color = color; for (int i = 0; resources != null && i < resources.Length; i++) { act.Resources[resources[i]] = count[i]; } embedActivities.Add(act); } else { if (Status == Status.Cancelled) { return; } if (CurrentActivity == null) { CurrentActivity = Activities[0]; } else { CurrentActivity = CurrentActivity.Next; } WaitForActivityStart(CurrentActivity); func(); } }