/** * This is either a request or response. Look up the handler * for it, and pass the packet to the handler */ public void HandleData(MemBlock p, ISender from, object state) { //Is it a request or reply? ReqrepType rt = (ReqrepType)((byte)p[0]); int idnum = NumberSerializer.ReadInt(p, 1); MemBlock rest = p.Slice(5); //Skip the type and the id if (rt == ReqrepType.Request || rt == ReqrepType.LossyRequest) { HandleRequest(rt, idnum, rest, from); } else if (rt == ReqrepType.Reply) { HandleReply(rt, idnum, rest, from); } else if (rt == ReqrepType.ReplyAck) { HandleReplyAck(rt, idnum, rest, from); } else if (rt == ReqrepType.RequestAck) { HandleRequestAck(rt, idnum, rest, from); } else if (rt == ReqrepType.Error) { HandleError(rt, idnum, rest, from); } }
/** * When we get a reply ack, we can remove the item from our cache, * we know the other guy got our reply */ protected void HandleReplyAck(ReqrepType rt, int idnum, MemBlock err_data, ISender ret_path) { RequestKey rk = new RequestKey(idnum, ret_path); lock ( _sync ) { /** * This is not completely safe, but probably fine. Consider the * case where: * A -(req)-> B * A timeout but B does get the req * A <-(rep)- B * A -(req)-> B (these cross in flight) * A -(repack)-> B * * but then the repack passes the req retransmission (out of order * delivery) * * This is unlikely, but we could improve it. * @todo improve the reply caching algorithm */ ReplyState rs = (ReplyState)_reply_cache[rk]; if (rs != null) { ReleaseReplyState(rs); } } }
protected void HandleRequestAck(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) { RequestState reqs; lock ( _sync ) { if (_req_state_table.TryGet(idnum, out reqs)) { if (reqs.AddAck(ret_path)) { /* * Let's look at how long it took to get this reply: */ TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate; if (ret_path is Edge) { _edge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _edge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS); } else { _nonedge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _nonedge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS); } } } } }
protected void HandleError(ReqrepType rt, int idnum, MemBlock err_data, ISender ret_path) { //Get the request: RequestState reqs; bool act; lock ( _sync ) { ///@todo, we might not want to stop listening after one error act = _req_state_table.TryTake(idnum, out reqs); } if (act) { #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepManager: {0}] Receiving error on request id: {1}, from: {2}", _info, idnum, ret_path); #endif ///@todo make sure we are checking that this ret_path makes sense for ///our request ReqrepError rrerr = (ReqrepError)err_data[0]; reqs.ReplyHandler.HandleError(this, idnum, rrerr, ret_path, reqs.UserState); } else { //We have already dealt with this Request } }
/** * @param sender how to send the request * @param reqt the type of request to make * @param data the data to encapsulate and send * @param reply the handler to handle the reply * @param state some state object to attach to this request * @return the identifier for this request * */ public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data, IReplyHandler reply, object state) { if (reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest) { throw new Exception("Not a request"); } TimeSpan timeout = sender is Edge ? _edge_reqtimeout : _nonedge_reqtimeout; RequestState rs = new RequestState(timeout, _acked_reqtimeout); rs.Sender = sender; rs.ReplyHandler = reply; rs.RequestType = reqt; rs.UserState = state; lock ( _sync ) { rs.RequestID = _req_state_table.GenerateID(rs); rs.Request = MakeRequest(reqt, rs.RequestID, data); } //Make sure that when we drop the lock, rs is totally initialized #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}", _info, rs.RequestID, sender); #endif try { rs.Send(); return(rs.RequestID); } catch { //Clean up: StopRequest(rs.RequestID, reply); throw; } }
protected ICopyable MakeRequest(ReqrepType rt, int next_rep, ICopyable data) { byte[] header = new byte[5]; header[0] = (byte)rt; NumberSerializer.WriteInt(next_rep, header, 1); MemBlock mb_header = MemBlock.Reference(header); return(new CopyList(_prefix, mb_header, data)); }
protected void HandleRequest(ReqrepType rt, int idnum, MemBlock rest, ISender retpath) { /** * Lets see if we have been asked this question before */ ReplyState rs = null; bool resend = false; #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepManager: {0}] Receiving request id: {1}, from: {2}", _info, idnum, retpath); #endif RequestKey rk = new RequestKey(idnum, retpath); lock ( _sync ) { rs = (ReplyState)_reply_cache[rk]; if (rs == null) { rs = GenerateReplyState(_prefix, rk); } else { resend = true; } } if (resend) { //This is an old request: rs.Resend(); } else { //This is a new request: try { _sub.Handle(rest, rs); } catch { lock ( _sync ) { ReleaseReplyState(rs); } //This didn't work out: try { MemBlock err_data = MemBlock.Reference( new byte[] { (byte)ReqrepError.HandlerFailure }); ICopyable reply = MakeRequest(ReqrepType.Error, idnum, err_data); retpath.Send(reply); } catch { //If this fails, we may think about logging. //The return path could fail, that's the only obvious exception ///@todo log exception } } } }
protected void HandleReply(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) { RequestState reqs; if (_req_state_table.TryGet(idnum, out reqs)) { IReplyHandler handler = null; lock ( _sync ) { if (reqs.AddReplier(ret_path)) { TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate; _to_mgr.AddReplySampleFor(reqs, ret_path, rtt); handler = reqs.ReplyHandler; } } /* * Now handle this reply */ if (null != handler) { MemBlock payload; PType pt = PType.Parse(rest, out payload); Statistics statistics = new Statistics(); statistics.SendCount = reqs.SendCount; #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepManager: {0}] Receiving reply on request id: {1}, from: {2}", _info, idnum, ret_path); #endif //Don't hold the lock while calling the ReplyHandler: bool continue_listening = handler.HandleReply(this, rt, idnum, pt, payload, ret_path, statistics, reqs.UserState); //the request has been served if (!continue_listening) { StopRequest(idnum, handler); } } } else { //We are ignoring this reply, it either makes no sense, or we have //already handled it } }
/** * @param sender how to send the request * @param reqt the type of request to make * @param data the data to encapsulate and send * @param reply the handler to handle the reply * @param state some state object to attach to this request * @return the identifier for this request * */ public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data, IReplyHandler reply, object state) { if (reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest) { throw new Exception("Not a request"); } TimeSpan timeout = _to_mgr.GetTimeOutFor(sender); RequestState rs = new RequestState(timeout, _to_mgr.AckedTimeOut); rs.Sender = sender; rs.ReplyHandler = reply; rs.RequestType = reqt; rs.UserState = state; lock ( _sync ) { rs.RequestID = _req_state_table.GenerateID(rs); rs.Request = MakeRequest(reqt, rs.RequestID, data); } //Make sure that when we drop the lock, rs is totally initialized #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}", _info, rs.RequestID, sender); #endif try { rs.Send(); return(rs.RequestID); } catch (SendException sx) { if (sx.IsTransient) { //I guess we will just try to resend again in the future: return(rs.RequestID); } else { //This is certainly going to fail, so fail now: StopRequest(rs.RequestID, reply); throw; } } catch { //Clean up: StopRequest(rs.RequestID, reply); throw; } }
protected void HandleRequestAck(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) { RequestState reqs; lock ( _sync ) { if (_req_state_table.TryGet(idnum, out reqs)) { if (reqs.AddAck(ret_path)) { /* * Let's look at how long it took to get this reply: */ TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate; _to_mgr.AddAckSampleFor(reqs, ret_path, rtt); } } } }
protected ICopyable MakeRequest(ReqrepType rt, int next_rep, ICopyable data) { byte[] header = new byte[ 5 ]; header[0] = (byte)rt; NumberSerializer.WriteInt( next_rep, header, 1 ); MemBlock mb_header = MemBlock.Reference(header); return new CopyList(_prefix, mb_header, data); }
/** * @param sender how to send the request * @param reqt the type of request to make * @param data the data to encapsulate and send * @param reply the handler to handle the reply * @param state some state object to attach to this request * @return the identifier for this request * */ public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data, IReplyHandler reply, object state) { if ( reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest ) { throw new Exception("Not a request"); } TimeSpan timeout = _to_mgr.GetTimeOutFor(sender); RequestState rs = new RequestState(timeout, _to_mgr.AckedTimeOut); rs.Sender = sender; rs.ReplyHandler = reply; rs.RequestType = reqt; rs.UserState = state; lock( _sync ) { rs.RequestID = _req_state_table.GenerateID(rs); rs.Request = MakeRequest(reqt, rs.RequestID, data); } //Make sure that when we drop the lock, rs is totally initialized #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}", _info, rs.RequestID, sender); #endif try { rs.Send(); return rs.RequestID; } catch(SendException sx) { if( sx.IsTransient ) { //I guess we will just try to resend again in the future: return rs.RequestID; } else { //This is certainly going to fail, so fail now: StopRequest(rs.RequestID, reply); throw; } } catch { //Clean up: StopRequest(rs.RequestID, reply); throw; } }
/** * When we get a reply ack, we can remove the item from our cache, * we know the other guy got our reply */ protected void HandleReplyAck(ReqrepType rt, int idnum, MemBlock err_data, ISender ret_path) { RequestKey rk = new RequestKey(idnum, ret_path); lock( _sync ) { /** * This is not completely safe, but probably fine. Consider the * case where: * A -(req)-> B * A timeout but B does get the req * A <-(rep)- B * A -(req)-> B (these cross in flight) * A -(repack)-> B * * but then the repack passes the req retransmission (out of order * delivery) * * This is unlikely, but we could improve it. * @todo improve the reply caching algorithm */ ReplyState rs = (ReplyState)_reply_cache[rk]; if( rs != null ) { ReleaseReplyState(rs); } } }
protected void HandleError(ReqrepType rt, int idnum, MemBlock err_data, ISender ret_path) { //Get the request: RequestState reqs; bool act; lock( _sync ) { ///@todo, we might not want to stop listening after one error act = _req_state_table.TryTake(idnum, out reqs); } if( act ) { #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepManager: {0}] Receiving error on request id: {1}, from: {2}", _info, idnum, ret_path); #endif ///@todo make sure we are checking that this ret_path makes sense for ///our request ReqrepError rrerr = (ReqrepError)err_data[0]; reqs.ReplyHandler.HandleError(this, idnum, rrerr, ret_path, reqs.UserState); } else { //We have already dealt with this Request } }
protected void HandleRequestAck(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) { RequestState reqs; lock( _sync ) { if( _req_state_table.TryGet(idnum, out reqs)) { if (reqs.AddAck(ret_path)) { /* * Let's look at how long it took to get this reply: */ TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate; _to_mgr.AddAckSampleFor(reqs, ret_path, rtt); } } } }
protected void HandleReply(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) { RequestState reqs; if( _req_state_table.TryGet(idnum, out reqs) ) { IReplyHandler handler = null; lock( _sync ) { if (reqs.AddReplier(ret_path)) { TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate; _to_mgr.AddReplySampleFor(reqs, ret_path, rtt); handler = reqs.ReplyHandler; } } /* * Now handle this reply */ if( null != handler ) { MemBlock payload; PType pt = PType.Parse(rest, out payload); Statistics statistics = new Statistics(); statistics.SendCount = reqs.SendCount; #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepManager: {0}] Receiving reply on request id: {1}, from: {2}", _info, idnum, ret_path); #endif //Don't hold the lock while calling the ReplyHandler: bool continue_listening = handler.HandleReply(this, rt, idnum, pt, payload, ret_path, statistics, reqs.UserState); //the request has been served if( !continue_listening ) { StopRequest(idnum, handler); } } } else { //We are ignoring this reply, it either makes no sense, or we have //already handled it } }
/** * @param sender how to send the request * @param reqt the type of request to make * @param data the data to encapsulate and send * @param reply the handler to handle the reply * @param state some state object to attach to this request * @return the identifier for this request * */ public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data, IReplyHandler reply, object state) { if ( reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest ) { throw new Exception("Not a request"); } RequestState rs = new RequestState(); rs.Sender = sender; rs.ReplyHandler = reply; rs.RequestType = reqt; rs.UserState = state; lock( _sync ) { //Get the index int next_req = 0; do { next_req = _rand.Next(); } while( _req_state_table.ContainsKey( next_req ) ); /* * Now we store the request */ rs.RequestID = next_req; rs.Request = MakeRequest(reqt, next_req, data); _req_state_table[ rs.RequestID ] = rs; } #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}", _info, rs.RequestID, sender); #endif try { rs.Send(); return rs.RequestID; } catch { //Clean up: StopRequest(rs.RequestID, reply); throw; } }
protected void HandleRequest(ReqrepType rt, int idnum, MemBlock rest, ISender retpath) { /** * Lets see if we have been asked this question before */ ReplyState rs = null; bool resend = false; #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepManager: {0}] Receiving request id: {1}, from: {2}", _info, idnum, retpath); #endif RequestKey rk = new RequestKey(idnum, retpath); lock( _sync ) { rs = (ReplyState)_reply_cache[rk]; if( rs == null ) { rs = GenerateReplyState(_prefix, rk); } else { resend = true; } } if( resend ) { //This is an old request: rs.Resend(); } else { //This is a new request: try { _sub.Handle(rest, rs); } catch { lock( _sync ) { ReleaseReplyState(rs); } //This didn't work out: try { MemBlock err_data = MemBlock.Reference( new byte[]{ (byte) ReqrepError.HandlerFailure } ); ICopyable reply = MakeRequest(ReqrepType.Error, idnum, err_data); retpath.Send(reply); } catch { //If this fails, we may think about logging. //The return path could fail, that's the only obvious exception ///@todo log exception } } } }
protected void HandleError(ReqrepType rt, int idnum, MemBlock err_data, ISender ret_path) { //Get the request: RequestState reqs = (RequestState)_req_state_table[idnum]; if( reqs != null ) { bool handle_error = false; lock( _sync ) { //Check to see if the request is still good, don't handle //the error twice: handle_error = _req_state_table.Contains(idnum); if( handle_error ) { ///@todo, we might not want to stop listening after one error _req_state_table.Remove(idnum); } } if( handle_error ) { #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepManager: {0}] Receiving error on request id: {1}, from: {2}", _info, idnum, ret_path); #endif ///@todo make sure we are checking that this ret_path makes sense for ///our request ReqrepError rrerr = (ReqrepError)err_data[0]; reqs.ReplyHandler.HandleError(this, idnum, rrerr, ret_path, reqs.UserState); } } else { //We have already dealt with this Request } }
protected void HandleReply(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) { RequestState reqs = (RequestState)_req_state_table[idnum]; if( reqs != null ) { IReplyHandler handler = null; lock( _sync ) { if (reqs.AddReplier(ret_path)) { TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate; /* * Let's look at how long it took to get this reply: */ if( reqs.GotAck ) { //Use more standard deviations for acked messages. We //just don't want to let it run forever. _acked_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _acked_rtt_stats, _MINIMUM_TIMEOUT, 3 *_STD_DEVS); } else if( ret_path is Edge ) { _edge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _edge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS); } else { _nonedge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _nonedge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS); } handler = reqs.ReplyHandler; } } /* * Now handle this reply */ if( null != handler ) { MemBlock payload; PType pt = PType.Parse(rest, out payload); Statistics statistics = new Statistics(); statistics.SendCount = reqs.SendCount; #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepManager: {0}] Receiving reply on request id: {1}, from: {2}", _info, idnum, ret_path); #endif //Don't hold the lock while calling the ReplyHandler: bool continue_listening = handler.HandleReply(this, rt, idnum, pt, payload, ret_path, statistics, reqs.UserState); //the request has been served if( !continue_listening ) { StopRequest(idnum, handler); } } } else { //We are ignoring this reply, it either makes no sense, or we have //already handled it } }
protected void HandleRequestAck(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) { RequestState reqs = (RequestState)_req_state_table[idnum]; if( reqs != null ) { lock( _sync ) { if (reqs.AddAck(ret_path)) { /* * Let's look at how long it took to get this reply: */ TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate; if( ret_path is Edge ) { _edge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _edge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS); } else { _nonedge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _nonedge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS); } } } } }
/** * @param sender how to send the request * @param reqt the type of request to make * @param data the data to encapsulate and send * @param reply the handler to handle the reply * @param state some state object to attach to this request * @return the identifier for this request * */ public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data, IReplyHandler reply, object state) { if ( reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest ) { throw new Exception("Not a request"); } TimeSpan timeout = sender is Edge ? _edge_reqtimeout : _nonedge_reqtimeout; RequestState rs = new RequestState(timeout, _acked_reqtimeout); rs.Sender = sender; rs.ReplyHandler = reply; rs.RequestType = reqt; rs.UserState = state; lock( _sync ) { rs.RequestID = _req_state_table.GenerateID(rs); rs.Request = MakeRequest(reqt, rs.RequestID, data); } //Make sure that when we drop the lock, rs is totally initialized #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}", _info, rs.RequestID, sender); #endif try { rs.Send(); return rs.RequestID; } catch { //Clean up: StopRequest(rs.RequestID, reply); throw; } }
protected void HandleReply(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) { RequestState reqs; if (_req_state_table.TryGet(idnum, out reqs)) { IReplyHandler handler = null; lock ( _sync ) { if (reqs.AddReplier(ret_path)) { TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate; /* * Let's look at how long it took to get this reply: */ if (reqs.GotAck) { //Use more standard deviations for acked messages. We //just don't want to let it run forever. _acked_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _acked_rtt_stats, _MINIMUM_TIMEOUT, 3 * _STD_DEVS); } else if (ret_path is Edge) { _edge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _edge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS); } else { _nonedge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds, _nonedge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS); } handler = reqs.ReplyHandler; } } /* * Now handle this reply */ if (null != handler) { MemBlock payload; PType pt = PType.Parse(rest, out payload); Statistics statistics = new Statistics(); statistics.SendCount = reqs.SendCount; #if REQREP_DEBUG Console.Error.WriteLine("[ReqrepManager: {0}] Receiving reply on request id: {1}, from: {2}", _info, idnum, ret_path); #endif //Don't hold the lock while calling the ReplyHandler: bool continue_listening = handler.HandleReply(this, rt, idnum, pt, payload, ret_path, statistics, reqs.UserState); //the request has been served if (!continue_listening) { StopRequest(idnum, handler); } } } else { //We are ignoring this reply, it either makes no sense, or we have //already handled it } }