Esempio n. 1
0
        public CentralizedProgressTracker(InternalGraphManager graphManager)
        {
            var centralizerProcessId = graphManager.Controller.Configuration.CentralizerProcessId;
            var centralizerThreadId  = graphManager.Controller.Configuration.CentralizerThreadId;

            var processes = graphManager.Controller.Configuration.Processes;
            var processid = graphManager.Controller.Configuration.ProcessID;

            Logging.Progress("Centralized progress tracker enabled, running on process {0} thread {1}", centralizerProcessId, centralizerThreadId);

            var context = new OpaqueTimeContext <Pointstamp>(graphManager.ContextManager.MakeRawContextForScope <Pointstamp>("progress context"));

            // construct aggregator stage and unconnected output
            var aggregatorPlacement = new SingleVertexPerProcessPlacement(processes, 0);
            var aggregatorStage     = new Stage <ProgressUpdateAggregator, Pointstamp>(aggregatorPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateAggregator(i, v), "Aggregator");
            var stream = aggregatorStage.NewOutput(shard => shard.Output);

            aggregatorStage.Materialize();
            this.aggregator = aggregatorStage.GetShard(processid);

            // construct centralizer stage and unconnected input and output
            var centralizerPlacement = new SingleVertexPlacement(centralizerProcessId, centralizerThreadId);
            var centralizer          = new Stage <ProgressUpdateCentralizer, Pointstamp>(centralizerPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateCentralizer(i, v, null), "Centralizer");
            var centralizerRecvPort  = centralizer.NewUnconnectedInput <Int64>(shard => shard.Input, null);
            var centralizerSendPort  = centralizer.NewOutput(shard => shard.Output, null);

            centralizer.Materialize();
            this.centralizer = (processid == centralizerProcessId) ? centralizer.GetShard(0) : null;

            // construct consumer stage and unconnected input
            var consumerPlacement = new SingleVertexPerProcessPlacement(processes, 0);
            var consumer          = new Stage <ProgressUpdateConsumer, Pointstamp>(consumerPlacement, context, Stage.OperatorType.Default, (i, v) => new Runtime.Progress.ProgressUpdateConsumer(i, v, this.aggregator), "Consumer");
            var consumerRecvPort  = consumer.NewUnconnectedInput(shard => shard.Input, null);

            consumer.Materialize();
            this.consumer = consumer.GetShard(processid);

            // connect centralizer to consumers with special progress channel
            var progressChannel = new ProgressChannel(centralizer.Placement.Count, this.consumer, centralizerSendPort.StageOutput, consumerRecvPort, graphManager.Controller, graphManager.AllocateNewGraphIdentifier());

            centralizerSendPort.StageOutput.AttachBundleToSender(progressChannel);

            // connect aggregators to centralizer with special centralized progress channel
            var centralizerChannel = new CentralizedProgressChannel(centralizer, stream.StageOutput, centralizerRecvPort, graphManager.Controller, graphManager.AllocateNewGraphIdentifier());

            stream.StageOutput.AttachBundleToSender(centralizerChannel);

            Logging.Progress("Centralized progress tracker initialization completed");
        }
Esempio n. 2
0
 public VertexInput(ProgressUpdateCentralizer op)
 {
     this.op = op;
 }