public ProgressChannel(int producerPlacementCount, ProgressUpdateConsumer consumerVertex, StageOutput <Update, Empty> stream, StageInput <Update, Empty> recvPort, InternalController controller, int channelId) { this.sendBundle = stream; this.recvBundle = recvPort; this.channelID = channelId; var computation = sendBundle.ForStage.InternalComputation; var recvFiber = this.recvBundle.GetPin(computation.Controller.Configuration.ProcessID); this.mailbox = new Mailbox(recvFiber.Vertex.Scheduler.State(computation).PostOffice, consumerVertex, this.channelID, consumerVertex.VertexId, producerPlacementCount); // recvFiber.Vertex.Scheduler.State(graphManager).PostOffice.RegisterMailbox(this.mailbox); this.postboxes = new Dictionary <int, Fiber>(); foreach (VertexLocation loc in sendBundle.ForStage.Placement) { if (loc.ProcessId == controller.Configuration.ProcessID) { this.postboxes[loc.VertexId] = new Fiber(this.channelID, loc.VertexId, this.sendBundle.GetFiber(loc.VertexId), this.mailbox, controller); } } if (controller.NetworkChannel != null) { controller.NetworkChannel.RegisterMailbox(this.mailbox); } Logging.Info("Allocated progress channel [{0}]: {1} -> {2}", this.channelID, sendBundle, recvBundle); }
/// <summary> /// Constructor /// </summary> public CentralizedProgressChannel(Stage <Runtime.Progress.ProgressUpdateCentralizer, Pointstamp> consumer, StageOutput <Int64, Pointstamp> stream, StageInput <Int64, Pointstamp> recvPort, InternalController controller, int channelId) { this.consumer = consumer; this.sendBundle = stream; // producer.Output; this.recvBundle = recvPort; // consumer.Input; this.postboxes = new Dictionary <int, Fiber>(); this.channelID = channelId; // Get the shard id and process id of the single consumer var consumerShardId = consumer.Placement.Single().VertexId; var consumerProcessId = consumer.Placement.Single().ProcessId; var graphManager = sendBundle.ForStage.InternalGraphManager; if (debug) { Console.Error.WriteLine(" IncastChannel create ProcessId = {0}", graphManager.Controller.Configuration.ProcessID); } var myProcessId = graphManager.Controller.Configuration.ProcessID; if (myProcessId == consumerProcessId) { if (debug) { Console.Error.WriteLine(" IncastChannel creating receive mailbox"); } VertexInput <Int64, Pointstamp> recvFiber = this.recvBundle.GetPin(consumerProcessId); this.mailbox = new Mailbox(recvFiber.Vertex.Scheduler.State(graphManager).PostOffice, consumer.GetShard(consumerShardId), this.channelID, consumerShardId); recvFiber.Vertex.Scheduler.State(graphManager).PostOffice.RegisterMailbox(this.mailbox); if (controller.NetworkChannel != null) { controller.NetworkChannel.RegisterMailbox(this.mailbox); } } foreach (VertexLocation loc in sendBundle.ForStage.Placement) { if (loc.ProcessId == sendBundle.ForStage.InternalGraphManager.Controller.Configuration.ProcessID) { if (debug) { Console.Error.WriteLine(" IncastChannel loc = {0}/{1}/{2}", loc.ProcessId, loc.VertexId, loc.ThreadId); } var postbox = new Fiber(this.channelID, loc.VertexId, this.sendBundle.GetFiber(loc.VertexId), this.mailbox, controller, consumerShardId, consumerProcessId); this.postboxes[loc.VertexId] = postbox; } } Logging.Info("Allocated incast channel [{0}]: {1} -> {2}", this.channelID, sendBundle, recvBundle); }
internal ReportingEgressStage(ITimeContext <T> externalContext) { receivers = new List <StageInput <string, IterationIn <T> > >(); intAggregator = null; doubleAggregator = null; this.stage = new Stage <ReportingEgressVertex <T>, T>(new TimeContext <T>(externalContext), Stage.OperatorType.IterationEgress, (i, v) => new ReportingEgressVertex <T>(i, v), "FixedPoint.ReportingEgress"); }
public PostOfficeChannel(StageOutput <S, T> sendBundle, StageInput <S, T> recvBundle, Action <S[], int[], int> routingHashcodeFunction, NetworkChannel networkChannel, int channelId, Channel.Flags flags) { if ((flags & Channel.Flags.SpillOnRecv) == Channel.Flags.SpillOnRecv) { throw new Exception("SpillingLocalMailbox not currently supported"); } this.sendBundle = sendBundle; this.recvBundle = recvBundle; this.channelID = channelId; var computation = sendBundle.ForStage.InternalComputation; // populate mailboxes; [proxy] destinations for sent messages. this.mailboxes = new Mailbox <S, T> [recvBundle.ForStage.Placement.Count]; foreach (VertexLocation loc in recvBundle.ForStage.Placement) { if (loc.ProcessId == computation.Controller.Configuration.ProcessID) { var postOffice = this.recvBundle.GetPin(loc.VertexId).Vertex.Scheduler.State(computation).PostOffice; var progressBuffer = new Runtime.Progress.ProgressUpdateBuffer <T>(this.ChannelId, this.recvBundle.GetPin(loc.VertexId).Vertex.Scheduler.State(computation).Producer); LocalMailbox <S, T> localMailbox = new LocalMailbox <S, T>(postOffice, this.recvBundle.GetPin(loc.VertexId), channelID, loc.VertexId, progressBuffer); this.mailboxes[loc.VertexId] = localMailbox; postOffice.RegisterMailbox(localMailbox); if (networkChannel != null) { networkChannel.RegisterMailbox(localMailbox); } } else { this.mailboxes[loc.VertexId] = new RemoteMailbox <S, T>(this.channelID, loc.ProcessId, loc.VertexId, sendBundle.ForStage.InternalComputation); } } // populate postboxes; collection points for each local worker. this.postboxes = new Dictionary <int, Postbox <S, T> >(); foreach (VertexLocation location in sendBundle.ForStage.Placement) { if (location.ProcessId == sendBundle.ForStage.InternalComputation.Controller.Configuration.ProcessID) { var progressBuffer = new Runtime.Progress.ProgressUpdateBuffer <T>(this.ChannelId, this.sendBundle.GetFiber(location.VertexId).Vertex.Scheduler.State(computation).Producer); // determine type of postbox to use. if (routingHashcodeFunction == null) { this.postboxes[location.VertexId] = new NoHashCodePostbox <S, T>(this.channelID, this.sendBundle.GetFiber(location.VertexId), this.recvBundle, this.mailboxes, networkChannel, progressBuffer); } else { this.postboxes[location.VertexId] = new BufferingPostbox <S, T>(this.channelID, this.sendBundle.GetFiber(location.VertexId), this.recvBundle, this.mailboxes, routingHashcodeFunction, networkChannel, progressBuffer); } } } }
public PostOfficeChannel(StageOutput <S, T> sendBundle, StageInput <S, T> recvBundle, Func <S, int> routingHashcodeFunction, NetworkChannel networkChannel, int channelId, Channel.Flags flags) { this.sendBundle = sendBundle; this.recvBundle = recvBundle; this.postboxes = new Dictionary <int, Postbox <S, T> >(); this.mailboxes = new Mailbox <S, T> [recvBundle.ForStage.Placement.Count]; this.channelID = channelId; var computation = sendBundle.ForStage.InternalComputation; foreach (VertexLocation loc in recvBundle.ForStage.Placement) { if (loc.ProcessId == computation.Controller.Configuration.ProcessID) { var postOffice = this.recvBundle.GetPin(loc.VertexId).Vertex.Scheduler.State(computation).PostOffice; LocalMailbox <S, T> localMailbox; var progressBuffer = new Runtime.Progress.ProgressUpdateBuffer <T>(this.ChannelId, this.recvBundle.GetPin(loc.VertexId).Vertex.Scheduler.State(computation).Producer); if ((flags & Channel.Flags.SpillOnRecv) == Channel.Flags.SpillOnRecv) { //localMailbox = new SpillingLocalMailbox<S, T>(postOffice, this.recvBundle.GetPin(loc.VertexId), channelID, loc.VertexId, progressBuffer); throw new Exception("SpillingLocalMailbox not currently supported"); } else { localMailbox = new LegacyLocalMailbox <S, T>(postOffice, this.recvBundle.GetPin(loc.VertexId), channelID, loc.VertexId, progressBuffer); } this.mailboxes[loc.VertexId] = localMailbox; postOffice.RegisterMailbox(localMailbox); if (networkChannel != null) { networkChannel.RegisterMailbox(localMailbox); } } else { this.mailboxes[loc.VertexId] = new RemoteMailbox <S, T>(this.channelID, loc.ProcessId, loc.VertexId, sendBundle.ForStage.InternalComputation); } } foreach (VertexLocation loc in sendBundle.ForStage.Placement) { if (loc.ProcessId == sendBundle.ForStage.InternalComputation.Controller.Configuration.ProcessID) { var progressBuffer = new Runtime.Progress.ProgressUpdateBuffer <T>(this.ChannelId, this.sendBundle.GetFiber(loc.VertexId).Vertex.Scheduler.State(computation).Producer); this.postboxes[loc.VertexId] = new Postbox <S, T>(this.channelID, this.sendBundle.GetFiber(loc.VertexId), this.recvBundle, this.mailboxes, routingHashcodeFunction, networkChannel, progressBuffer); } } }
internal Feedback(ITimeContext <IterationIn <TTime> > context, Expression <Func <TRecord, int> > partitionedBy, int maxIterations) { this.stage = new Stage <AdvanceVertex <TRecord, TTime>, IterationIn <TTime> >(new TimeContext <IterationIn <TTime> >(context), Stage.OperatorType.IterationAdvance, (i, v) => new AdvanceVertex <TRecord, TTime>(i, v, maxIterations), "Iterate.Advance"); this.input = this.stage.NewUnconnectedInput((message, vertex) => vertex.OnReceive(message), partitionedBy); this.output = this.stage.NewOutput(vertex => vertex.VertexOutput, partitionedBy); this.PartitionedBy = partitionedBy; this.MaxIterations = maxIterations; }
internal Feedback(ITimeContext <IterationIn <T> > context, Expression <Func <R, int> > partitionedBy, int maxIterations) { this.stage = new Stage <AdvanceShard <R, T>, IterationIn <T> >(new OpaqueTimeContext <IterationIn <T> >(context), Stage.OperatorType.IterationAdvance, (i, v) => new AdvanceShard <R, T>(i, v, maxIterations), "Iterate.Advance"); this.input = this.stage.NewUnconnectedInput((message, shard) => shard.MessageReceived(message), partitionedBy); this.output = this.stage.NewOutput(shard => shard.ShardOutput, partitionedBy); this.PartitionedBy = partitionedBy; this.MaxIterations = maxIterations; }
public Postbox(int channelID, VertexOutput <S, T> sender, StageInput <S, T> receiverBundle, Mailbox <S, T>[] mailboxes, Action <S[], int[], int> routingHashcodeFunction, NetworkChannel networkChannel) { this.threadindex = sender.Vertex.Scheduler.Index; this.channelID = channelID; this.sender = sender; this.vertexid = sender.Vertex.VertexId; this.receiverBundle = receiverBundle; this.routingHashcodesFunction = routingHashcodeFunction; this.networkChannel = networkChannel; this.mailboxes = mailboxes; this.returnAddress = new ReturnAddress(this.ProcessID, this.vertexid, this.threadindex); }
public Postbox(int channelID, VertexOutput <S, T> sender, StageInput <S, T> receiverBundle, Mailbox <S, T>[] mailboxes, Func <S, int> routingHashcodeFunction, NetworkChannel networkChannel) { this.threadindex = sender.Vertex.Scheduler.Index; this.channelID = channelID; this.sender = sender; this.shardid = sender.Vertex.VertexId; this.receiverBundle = receiverBundle; this.routingHashcodeFunction = routingHashcodeFunction; this.networkChannel = networkChannel; this.mailboxes = mailboxes; //this.sendSequenceNumbers = new int[Naiad.Processes, this.receiverBundle.LocalParallelism]; }
public BufferingPostbox(int channelID, VertexOutput <S, T> sender, StageInput <S, T> receiverBundle, Mailbox <S, T>[] mailboxes, Action <S[], int[], int> routingHashcodeFunction, NetworkChannel networkChannel, Runtime.Progress.ProgressUpdateBuffer <T> progressBuffer) : base(channelID, sender, receiverBundle, mailboxes, routingHashcodeFunction, networkChannel, progressBuffer) { if (((this.mailboxes.Length - 1) & this.mailboxes.Length) == 0) { this.mask = this.mailboxes.Length - 1; } else { this.mask = -1; } this.Buffers = new Message <S, T> [this.mailboxes.Length]; }
/// <summary> /// Constructor /// </summary> public CentralizedProgressChannel(Stage <ProgressUpdateCentralizer, Empty> consumer, StageOutput <Update, Empty> stream, StageInput <Update, Empty> recvPort, InternalController controller, int channelId) { this.consumer = consumer; this.sendBundle = stream; // producer.Output; this.recvBundle = recvPort; // consumer.Input; this.postboxes = new Dictionary <int, Fiber>(); this.channelID = channelId; // Get the vertex id and process id of the single consumer var consumerVertexId = consumer.Placement.Single().VertexId; var consumerProcessId = consumer.Placement.Single().ProcessId; var computation = sendBundle.ForStage.InternalComputation; var myProcessId = computation.Controller.Configuration.ProcessID; if (myProcessId == consumerProcessId) { VertexInput <Update, Empty> recvFiber = this.recvBundle.GetPin(consumerProcessId); this.mailbox = new Mailbox(recvFiber.Vertex.Scheduler.State(computation).PostOffice, consumer.GetVertex(consumerVertexId), this.channelID, consumerVertexId); //recvFiber.Vertex.Scheduler.State(computation).PostOffice.RegisterMailbox(this.mailbox); if (controller.NetworkChannel != null) { controller.NetworkChannel.RegisterMailbox(this.mailbox); } } foreach (VertexLocation loc in sendBundle.ForStage.Placement) { if (loc.ProcessId == sendBundle.ForStage.InternalComputation.Controller.Configuration.ProcessID) { var postbox = new Fiber(this.channelID, loc.VertexId, this.sendBundle.GetFiber(loc.VertexId), this.mailbox, controller, consumerVertexId, consumerProcessId); this.postboxes[loc.VertexId] = postbox; } } Logging.Info("Allocated CentralizedProgressChannel [{0}]: {1} -> {2}", this.channelID, sendBundle, recvBundle); NaiadTracing.Trace.ChannelInfo(ChannelId, SourceStage.StageId, DestinationStage.StageId, true, true); }
public PipelineChannel(StageOutput <S, T> sender, StageInput <S, T> receiver, int channelId) { this.sender = sender; this.receiver = receiver; this.channelId = channelId; this.subChannels = new Dictionary <int, Fiber>(); foreach (VertexLocation loc in sender.ForStage.Placement) { if (loc.ProcessId == sender.ForStage.InternalComputation.Controller.Configuration.ProcessID) { this.subChannels[loc.VertexId] = new Fiber(this, receiver.GetPin(loc.VertexId), loc.VertexId); } } }
public Postbox(int channelID, VertexOutput <S, T> sender, StageInput <S, T> receiverBundle, Mailbox <S, T>[] mailboxes, Func <S, int> routingHashcodeFunction, NetworkChannel networkChannel, Runtime.Progress.ProgressUpdateBuffer <T> progressBuffer) : this(channelID, sender, receiverBundle, mailboxes, routingHashcodeFunction, networkChannel) { this.progressBuffer = progressBuffer; }
internal static bool PartitionedEquivalently <S, T>(StageOutput <S, T> sender, StageInput <S, T> receiver) where T : Time <T> { return(sender.ForStage.Placement.Equals(receiver.ForStage.Placement) && sender.PartitionedBy != null && receiver.PartitionedBy != null && Utilities.ExpressionComparer.Instance.Equals(sender.PartitionedBy, receiver.PartitionedBy)); }
internal static bool Pipelineable <S, T>(StageOutput <S, T> sender, StageInput <S, T> receiver, Action <S[], int[], int> key) where T : Time <T> { return(sender.ForStage.Placement.Equals(receiver.ForStage.Placement) && key == null); }
public NoHashCodePostbox(int channelID, VertexOutput <S, T> sender, StageInput <S, T> receiverBundle, Mailbox <S, T>[] mailboxes, NetworkChannel networkChannel, Runtime.Progress.ProgressUpdateBuffer <T> progressBuffer) : base(channelID, sender, receiverBundle, mailboxes, null, networkChannel, progressBuffer) { this.destination = this.vertexid % this.mailboxes.Length; }
public void ConnectDoubleAggregator(Stream <Pair <string, ReportingRecord <double> >, IterationIn <T> > sender) { System.Diagnostics.Debug.Assert(doubleAggregator == null); doubleAggregator = stage.NewSurprisingTimeTypeInput(sender, vertex => new ActionReceiver <Pair <string, ReportingRecord <double> >, IterationIn <T> >(vertex, (m, p) => vertex.ForwardDoubleAggregate(m, p)), null); }
public void ConnectIntAggregator(Stream <Pair <string, ReportingRecord <Int64> >, IterationIn <T> > sender) { System.Diagnostics.Debug.Assert(intAggregator == null); intAggregator = stage.NewSurprisingTimeTypeInput(sender, shard => new ActionReceiver <Pair <string, ReportingRecord <Int64> >, IterationIn <T> >(shard, (m, p) => shard.ForwardIntAggregate(m, p)), null); }