public DistributedProgressTracker(InternalComputation internalComputation) { var processes = internalComputation.Controller.Configuration.Processes; var processid = internalComputation.Controller.Configuration.ProcessID; var context = new TimeContext <Empty>(internalComputation.ContextManager.MakeRawContextForScope <Empty>("progress context")); // construct aggregator stage with unconnected output var aggregatorPlacement = new Placement.SingleVertexPerProcess(processes, 0); var aggregator = new Stage <ProgressUpdateAggregator, Empty>(aggregatorPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateAggregator(i, v), "Aggregator"); var stream = aggregator.NewOutput(vertex => vertex.Output); aggregator.Materialize(); this.aggregator = aggregator.GetVertex(processid); // construct consumer stage with unconnected input var consumerPlacement = new Placement.SingleVertexPerProcess(processes, 0); var consumer = new Stage <ProgressUpdateConsumer, Empty>(consumerPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateConsumer(i, v, this.aggregator), "Consumer"); var recvPort = consumer.NewUnconnectedInput(vertex => vertex.Input, null); consumer.Materialize(); this.consumer = consumer.GetVertex(processid); // connect aggregators to consumers with special progress channel this.progressChannel = new ProgressChannel(aggregatorPlacement.Count, this.consumer, stream.StageOutput, recvPort, internalComputation.Controller, internalComputation.AllocateNewGraphIdentifier()); stream.StageOutput.AttachBundleToSender(this.progressChannel); Logging.Progress("Distributed progress tracker enabled"); }
internal ProgressUpdateConsumer(int index, Stage <Empty> stage, ProgressUpdateAggregator aggregator) : base(index, stage) { this.Aggregator = aggregator; this.PCS = new PointstampCountSet(this.Stage.InternalComputation.Reachability); }
internal ProgressUpdateCentralizer(int index, Stage <Empty> stage, ProgressUpdateAggregator aggregator) : base(index, stage) { this.Aggregator = aggregator; this.Output = new VertexOutputBuffer <Update, Empty>(this); this.PCS = new PointstampCountSet(this.Stage.InternalComputation.Reachability); }
public CentralizedProgressTracker(InternalComputation internalComputation) { var centralizerProcessId = internalComputation.Controller.Configuration.CentralizerProcessId; var centralizerThreadId = internalComputation.Controller.Configuration.CentralizerThreadId; var processes = internalComputation.Controller.Configuration.Processes; var processid = internalComputation.Controller.Configuration.ProcessID; Logging.Progress("Centralized progress tracker enabled, running on process {0} thread {1}", centralizerProcessId, centralizerThreadId); var context = new TimeContext <Empty>(internalComputation.ContextManager.MakeRawContextForScope <Empty>("progress context")); // construct aggregator stage and unconnected output var aggregatorPlacement = new Placement.SingleVertexPerProcess(processes, 0); var aggregatorStage = new Stage <ProgressUpdateAggregator, Empty>(aggregatorPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateAggregator(i, v), "Aggregator"); var stream = aggregatorStage.NewOutput(vertex => vertex.Output); aggregatorStage.Materialize(); this.aggregator = aggregatorStage.GetVertex(processid); // construct centralizer stage and unconnected input and output var centralizerPlacement = new Placement.SingleVertex(centralizerProcessId, centralizerThreadId); var centralizer = new Stage <ProgressUpdateCentralizer, Empty>(centralizerPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateCentralizer(i, v, null), "Centralizer"); var centralizerRecvPort = centralizer.NewUnconnectedInput <Update>(vertex => vertex.Input, null); var centralizerSendPort = centralizer.NewOutput(vertex => vertex.Output, null); centralizer.Materialize(); this.centralizer = (processid == centralizerProcessId) ? centralizer.GetVertex(0) : null; // construct consumer stage and unconnected input var consumerPlacement = new Placement.SingleVertexPerProcess(processes, 0); var consumer = new Stage <ProgressUpdateConsumer, Empty>(consumerPlacement, context, Stage.OperatorType.Default, (i, v) => new Runtime.Progress.ProgressUpdateConsumer(i, v, this.aggregator), "Consumer"); var consumerRecvPort = consumer.NewUnconnectedInput(vertex => vertex.Input, null); consumer.Materialize(); this.consumer = consumer.GetVertex(processid); // connect centralizer to consumers with special progress channel var progressChannel = new ProgressChannel(centralizer.Placement.Count, this.consumer, centralizerSendPort.StageOutput, consumerRecvPort, internalComputation.Controller, internalComputation.AllocateNewGraphIdentifier()); centralizerSendPort.StageOutput.AttachBundleToSender(progressChannel); // connect aggregators to centralizer with special centralized progress channel var centralizerChannel = new CentralizedProgressChannel(centralizer, stream.StageOutput, centralizerRecvPort, internalComputation.Controller, internalComputation.AllocateNewGraphIdentifier()); stream.StageOutput.AttachBundleToSender(centralizerChannel); Logging.Progress("Centralized progress tracker initialization completed"); }
internal ProgressUpdateProducer(InternalComputation manager, ProgressUpdateAggregator aggregator) { this.LocalPCS = new PointstampCountSet(manager.Reachability); this.Aggregator = aggregator; NaiadTracing.Trace.LockInfo(this, "Producer lock"); }
internal ProgressUpdateProducer(InternalComputation manager, ProgressUpdateAggregator aggregator) { this.LocalPCS = new PointstampCountSet(manager.Reachability); this.Aggregator = aggregator; }