public DistributedProgressTracker(InternalGraphManager graphManager)
{
var processes = graphManager.Controller.Configuration.Processes;
var processid = graphManager.Controller.Configuration.ProcessID;
var context = new OpaqueTimeContext <Pointstamp>(graphManager.ContextManager.MakeRawContextForScope <Pointstamp>("progress context"));
// construct aggregator stage with unconnected output
var aggregatorPlacement = new SingleVertexPerProcessPlacement(processes, 0);
var aggregator = new Stage <ProgressUpdateAggregator, Pointstamp>(aggregatorPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateAggregator(i, v), "Aggregator");
var stream = aggregator.NewOutput(shard => shard.Output);
aggregator.Materialize();
this.aggregator = aggregator.GetShard(processid);
// construct consumer stage with unconnected input
var consumerPlacement = new SingleVertexPerProcessPlacement(processes, 0);
var consumer = new Stage <ProgressUpdateConsumer, Pointstamp>(consumerPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateConsumer(i, v, this.aggregator), "Consumer");
var recvPort = consumer.NewUnconnectedInput(shard => shard.Input, null);
consumer.Materialize();
this.consumer = consumer.GetShard(processid);
// connect aggregators to consumers with special progress channel
this.progressChannel = new ProgressChannel(aggregatorPlacement.Count, this.consumer, stream.StageOutput, recvPort, graphManager.Controller, graphManager.AllocateNewGraphIdentifier());
stream.StageOutput.AttachBundleToSender(this.progressChannel);
Logging.Progress("Distributed progress tracker enabled");
}
public SubChannel(ProgressChannel root, int id, int level)
{
Contracts.AssertValue(root);
Contracts.Assert(level >= 0);
_root = root;
_id = id;
_level = level;
_headerAndAction = Tuple.Create <ProgressHeader, Action <IProgressEntry> >(new ProgressHeader(null), null);
Start();
}
public void Log(ProgressChannel source, ProgressEvent.EventKind kind, ProgressEntry entry)
{
_ectx.AssertValue(source);
_ectx.AssertValueOrNull(entry);
if (kind == ProgressEvent.EventKind.Start)
{
_ectx.Assert(entry == null);
lock (_lock)
{
// Figure out an appropriate name.
int i = 1;
var name = source.Name;
string nameCandidate = name;
while (!_namesUsed.Add(nameCandidate))
{
i++;
nameCandidate = string.Format("{0} #{1}", name, i);
}
var newInfo = new CalculationInfo(++_index, nameCandidate, source);
_infos.Add(newInfo);
_pendingEvents.Enqueue(new ProgressEvent(newInfo.Index, newInfo.Name, newInfo.StartTime, ProgressEvent.EventKind.Start));
return;
}
}
// Not a start event, so we won't modify the _infos.
CalculationInfo info;
lock (_lock)
{
info = _infos.FirstOrDefault(x => x.Channel == source);
if (info == null)
{
throw _ectx.Except("Event sent after the calculation lifetime expired.");
}
}
switch (kind)
{
case ProgressEvent.EventKind.Stop:
_ectx.Assert(entry == null);
info.IsFinished = true;
_pendingEvents.Enqueue(new ProgressEvent(info.Index, info.Name, info.StartTime, ProgressEvent.EventKind.Stop));
break;
default:
_ectx.Assert(entry != null);
_ectx.Assert(kind == ProgressEvent.EventKind.Progress);
_ectx.Assert(!info.IsFinished);
_pendingEvents.Enqueue(new ProgressEvent(info.Index, info.Name, info.StartTime, entry));
break;
}
}
public CalculationInfo(int index, string name, ProgressChannel channel)
{
Contracts.Assert(index > 0);
Contracts.AssertNonEmpty(name);
Contracts.AssertValue(channel);
Index = index;
Name = name;
PendingCheckpoints = new ConcurrentQueue <KeyValuePair <DateTime, ProgressEntry> >();
StartTime = DateTime.UtcNow;
Channel = channel;
}
public CentralizedProgressTracker(InternalGraphManager graphManager)
{
var centralizerProcessId = graphManager.Controller.Configuration.CentralizerProcessId;
var centralizerThreadId = graphManager.Controller.Configuration.CentralizerThreadId;
var processes = graphManager.Controller.Configuration.Processes;
var processid = graphManager.Controller.Configuration.ProcessID;
Logging.Progress("Centralized progress tracker enabled, running on process {0} thread {1}", centralizerProcessId, centralizerThreadId);
var context = new OpaqueTimeContext <Pointstamp>(graphManager.ContextManager.MakeRawContextForScope <Pointstamp>("progress context"));
// construct aggregator stage and unconnected output
var aggregatorPlacement = new SingleVertexPerProcessPlacement(processes, 0);
var aggregatorStage = new Stage <ProgressUpdateAggregator, Pointstamp>(aggregatorPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateAggregator(i, v), "Aggregator");
var stream = aggregatorStage.NewOutput(shard => shard.Output);
aggregatorStage.Materialize();
this.aggregator = aggregatorStage.GetShard(processid);
// construct centralizer stage and unconnected input and output
var centralizerPlacement = new SingleVertexPlacement(centralizerProcessId, centralizerThreadId);
var centralizer = new Stage <ProgressUpdateCentralizer, Pointstamp>(centralizerPlacement, context, Stage.OperatorType.Default, (i, v) => new ProgressUpdateCentralizer(i, v, null), "Centralizer");
var centralizerRecvPort = centralizer.NewUnconnectedInput <Int64>(shard => shard.Input, null);
var centralizerSendPort = centralizer.NewOutput(shard => shard.Output, null);
centralizer.Materialize();
this.centralizer = (processid == centralizerProcessId) ? centralizer.GetShard(0) : null;
// construct consumer stage and unconnected input
var consumerPlacement = new SingleVertexPerProcessPlacement(processes, 0);
var consumer = new Stage <ProgressUpdateConsumer, Pointstamp>(consumerPlacement, context, Stage.OperatorType.Default, (i, v) => new Runtime.Progress.ProgressUpdateConsumer(i, v, this.aggregator), "Consumer");
var consumerRecvPort = consumer.NewUnconnectedInput(shard => shard.Input, null);
consumer.Materialize();
this.consumer = consumer.GetShard(processid);
// connect centralizer to consumers with special progress channel
var progressChannel = new ProgressChannel(centralizer.Placement.Count, this.consumer, centralizerSendPort.StageOutput, consumerRecvPort, graphManager.Controller, graphManager.AllocateNewGraphIdentifier());
centralizerSendPort.StageOutput.AttachBundleToSender(progressChannel);
// connect aggregators to centralizer with special centralized progress channel
var centralizerChannel = new CentralizedProgressChannel(centralizer, stream.StageOutput, centralizerRecvPort, graphManager.Controller, graphManager.AllocateNewGraphIdentifier());
stream.StageOutput.AttachBundleToSender(centralizerChannel);
Logging.Progress("Centralized progress tracker initialization completed");
}
