public DistributedProgressTracker(InternalComputation internalComputation)
{
var processes = internalComputation.Controller.Configuration.Processes;
var processid = internalComputation.Controller.Configuration.ProcessID;
var context = new TimeContext <Empty>(internalComputation.ContextManager.MakeRawContextForScope <Empty>("progress context"));
// construct aggregator stage with unconnected output
var aggregatorPlacement = new Placement.SingleVertexPerProcess(processes, 0);
var aggregator = new Stage <ProgressUpdateAggregator, Empty>(aggregatorPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateAggregator(i, v), "Aggregator");
var stream = aggregator.NewOutput(vertex => vertex.Output);
aggregator.Materialize();
this.aggregator = aggregator.GetVertex(processid);
// construct consumer stage with unconnected input
var consumerPlacement = new Placement.SingleVertexPerProcess(processes, 0);
var consumer = new Stage <ProgressUpdateConsumer, Empty>(consumerPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateConsumer(i, v, this.aggregator), "Consumer");
var recvPort = consumer.NewUnconnectedInput(vertex => vertex.Input, null);
consumer.Materialize();
this.consumer = consumer.GetVertex(processid);
// connect aggregators to consumers with special progress channel
this.progressChannel = new ProgressChannel(aggregatorPlacement.Count, this.consumer, stream.StageOutput, recvPort, internalComputation.Controller, internalComputation.AllocateNewGraphIdentifier());
stream.StageOutput.AttachBundleToSender(this.progressChannel);
Logging.Progress("Distributed progress tracker enabled");
}
public CentralizedProgressTracker(InternalComputation internalComputation)
{
var centralizerProcessId = internalComputation.Controller.Configuration.CentralizerProcessId;
var centralizerThreadId = internalComputation.Controller.Configuration.CentralizerThreadId;
var processes = internalComputation.Controller.Configuration.Processes;
var processid = internalComputation.Controller.Configuration.ProcessID;
Logging.Progress("Centralized progress tracker enabled, running on process {0} thread {1}", centralizerProcessId, centralizerThreadId);
var context = new TimeContext <Empty>(internalComputation.ContextManager.MakeRawContextForScope <Empty>("progress context"));
// construct aggregator stage and unconnected output
var aggregatorPlacement = new Placement.SingleVertexPerProcess(processes, 0);
var aggregatorStage = new Stage <ProgressUpdateAggregator, Empty>(aggregatorPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateAggregator(i, v), "Aggregator");
var stream = aggregatorStage.NewOutput(vertex => vertex.Output);
aggregatorStage.Materialize();
this.aggregator = aggregatorStage.GetVertex(processid);
// construct centralizer stage and unconnected input and output
var centralizerPlacement = new Placement.SingleVertex(centralizerProcessId, centralizerThreadId);
var centralizer = new Stage <ProgressUpdateCentralizer, Empty>(centralizerPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateCentralizer(i, v, null), "Centralizer");
var centralizerRecvPort = centralizer.NewUnconnectedInput <Update>(vertex => vertex.Input, null);
var centralizerSendPort = centralizer.NewOutput(vertex => vertex.Output, null);
centralizer.Materialize();
this.centralizer = (processid == centralizerProcessId) ? centralizer.GetVertex(0) : null;
// construct consumer stage and unconnected input
var consumerPlacement = new Placement.SingleVertexPerProcess(processes, 0);
var consumer = new Stage <ProgressUpdateConsumer, Empty>(consumerPlacement, context, Stage.OperatorType.Default, (i, v) => new Runtime.Progress.ProgressUpdateConsumer(i, v, this.aggregator), "Consumer");
var consumerRecvPort = consumer.NewUnconnectedInput(vertex => vertex.Input, null);
consumer.Materialize();
this.consumer = consumer.GetVertex(processid);
// connect centralizer to consumers with special progress channel
var progressChannel = new ProgressChannel(centralizer.Placement.Count, this.consumer, centralizerSendPort.StageOutput, consumerRecvPort, internalComputation.Controller, internalComputation.AllocateNewGraphIdentifier());
centralizerSendPort.StageOutput.AttachBundleToSender(progressChannel);
// connect aggregators to centralizer with special centralized progress channel
var centralizerChannel = new CentralizedProgressChannel(centralizer, stream.StageOutput, centralizerRecvPort, internalComputation.Controller, internalComputation.AllocateNewGraphIdentifier());
stream.StageOutput.AttachBundleToSender(centralizerChannel);
Logging.Progress("Centralized progress tracker initialization completed");
}
public VertexInput(ProgressUpdateConsumer op)
{
this.op = op;
}