/**
* This is either a request or response. Look up the handler
* for it, and pass the packet to the handler
*/
public void HandleData(MemBlock p, ISender from, object state)
{
//Is it a request or reply?
ReqrepType rt = (ReqrepType)((byte)p[0]);
int idnum = NumberSerializer.ReadInt(p, 1);
MemBlock rest = p.Slice(5); //Skip the type and the id
if (rt == ReqrepType.Request || rt == ReqrepType.LossyRequest)
{
HandleRequest(rt, idnum, rest, from);
}
else if (rt == ReqrepType.Reply)
{
HandleReply(rt, idnum, rest, from);
}
else if (rt == ReqrepType.ReplyAck)
{
HandleReplyAck(rt, idnum, rest, from);
}
else if (rt == ReqrepType.RequestAck)
{
HandleRequestAck(rt, idnum, rest, from);
}
else if (rt == ReqrepType.Error)
{
HandleError(rt, idnum, rest, from);
}
}
/**
* When we get a reply ack, we can remove the item from our cache,
* we know the other guy got our reply
*/
protected void HandleReplyAck(ReqrepType rt, int idnum,
MemBlock err_data, ISender ret_path)
{
RequestKey rk = new RequestKey(idnum, ret_path);
lock ( _sync ) {
/**
* This is not completely safe, but probably fine. Consider the
* case where:
* A -(req)-> B
* A timeout but B does get the req
* A <-(rep)- B
* A -(req)-> B (these cross in flight)
* A -(repack)-> B
*
* but then the repack passes the req retransmission (out of order
* delivery)
*
* This is unlikely, but we could improve it.
* @todo improve the reply caching algorithm
*/
ReplyState rs = (ReplyState)_reply_cache[rk];
if (rs != null)
{
ReleaseReplyState(rs);
}
}
}
protected void HandleRequestAck(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path)
{
RequestState reqs;
lock ( _sync ) {
if (_req_state_table.TryGet(idnum, out reqs))
{
if (reqs.AddAck(ret_path))
{
/*
* Let's look at how long it took to get this reply:
*/
TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate;
if (ret_path is Edge)
{
_edge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_edge_rtt_stats,
_MINIMUM_TIMEOUT, _STD_DEVS);
}
else
{
_nonedge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_nonedge_rtt_stats,
_MINIMUM_TIMEOUT, _STD_DEVS);
}
}
}
}
}
protected void HandleError(ReqrepType rt, int idnum,
MemBlock err_data, ISender ret_path)
{
//Get the request:
RequestState reqs;
bool act;
lock ( _sync ) {
///@todo, we might not want to stop listening after one error
act = _req_state_table.TryTake(idnum, out reqs);
}
if (act)
{
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepManager: {0}] Receiving error on request id: {1}, from: {2}",
_info, idnum, ret_path);
#endif
///@todo make sure we are checking that this ret_path makes sense for
///our request
ReqrepError rrerr = (ReqrepError)err_data[0];
reqs.ReplyHandler.HandleError(this, idnum, rrerr, ret_path, reqs.UserState);
}
else
{
//We have already dealt with this Request
}
}
/**
* @param sender how to send the request
* @param reqt the type of request to make
* @param data the data to encapsulate and send
* @param reply the handler to handle the reply
* @param state some state object to attach to this request
* @return the identifier for this request
*
*/
public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data,
IReplyHandler reply, object state)
{
if (reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest)
{
throw new Exception("Not a request");
}
TimeSpan timeout = sender is Edge ? _edge_reqtimeout : _nonedge_reqtimeout;
RequestState rs = new RequestState(timeout, _acked_reqtimeout);
rs.Sender = sender;
rs.ReplyHandler = reply;
rs.RequestType = reqt;
rs.UserState = state;
lock ( _sync ) {
rs.RequestID = _req_state_table.GenerateID(rs);
rs.Request = MakeRequest(reqt, rs.RequestID, data);
}
//Make sure that when we drop the lock, rs is totally initialized
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}",
_info, rs.RequestID, sender);
#endif
try {
rs.Send();
return(rs.RequestID);
}
catch {
//Clean up:
StopRequest(rs.RequestID, reply);
throw;
}
}
protected ICopyable MakeRequest(ReqrepType rt, int next_rep, ICopyable data)
{
byte[] header = new byte[5];
header[0] = (byte)rt;
NumberSerializer.WriteInt(next_rep, header, 1);
MemBlock mb_header = MemBlock.Reference(header);
return(new CopyList(_prefix, mb_header, data));
}
protected void HandleRequest(ReqrepType rt, int idnum,
MemBlock rest, ISender retpath)
{
/**
* Lets see if we have been asked this question before
*/
ReplyState rs = null;
bool resend = false;
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepManager: {0}] Receiving request id: {1}, from: {2}",
_info, idnum, retpath);
#endif
RequestKey rk = new RequestKey(idnum, retpath);
lock ( _sync ) {
rs = (ReplyState)_reply_cache[rk];
if (rs == null)
{
rs = GenerateReplyState(_prefix, rk);
}
else
{
resend = true;
}
}
if (resend)
{
//This is an old request:
rs.Resend();
}
else
{
//This is a new request:
try {
_sub.Handle(rest, rs);
}
catch {
lock ( _sync ) {
ReleaseReplyState(rs);
}
//This didn't work out:
try {
MemBlock err_data = MemBlock.Reference(
new byte[] { (byte)ReqrepError.HandlerFailure });
ICopyable reply = MakeRequest(ReqrepType.Error, idnum, err_data);
retpath.Send(reply);
}
catch {
//If this fails, we may think about logging.
//The return path could fail, that's the only obvious exception
///@todo log exception
}
}
}
}
protected void HandleReply(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path)
{
RequestState reqs;
if (_req_state_table.TryGet(idnum, out reqs))
{
IReplyHandler handler = null;
lock ( _sync ) {
if (reqs.AddReplier(ret_path))
{
TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate;
_to_mgr.AddReplySampleFor(reqs, ret_path, rtt);
handler = reqs.ReplyHandler;
}
}
/*
* Now handle this reply
*/
if (null != handler)
{
MemBlock payload;
PType pt = PType.Parse(rest, out payload);
Statistics statistics = new Statistics();
statistics.SendCount = reqs.SendCount;
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepManager: {0}] Receiving reply on request id: {1}, from: {2}",
_info, idnum, ret_path);
#endif
//Don't hold the lock while calling the ReplyHandler:
bool continue_listening = handler.HandleReply(this, rt, idnum, pt, payload,
ret_path, statistics, reqs.UserState);
//the request has been served
if (!continue_listening)
{
StopRequest(idnum, handler);
}
}
}
else
{
//We are ignoring this reply, it either makes no sense, or we have
//already handled it
}
}
/**
* @param sender how to send the request
* @param reqt the type of request to make
* @param data the data to encapsulate and send
* @param reply the handler to handle the reply
* @param state some state object to attach to this request
* @return the identifier for this request
*
*/
public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data,
IReplyHandler reply, object state)
{
if (reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest)
{
throw new Exception("Not a request");
}
TimeSpan timeout = _to_mgr.GetTimeOutFor(sender);
RequestState rs = new RequestState(timeout, _to_mgr.AckedTimeOut);
rs.Sender = sender;
rs.ReplyHandler = reply;
rs.RequestType = reqt;
rs.UserState = state;
lock ( _sync ) {
rs.RequestID = _req_state_table.GenerateID(rs);
rs.Request = MakeRequest(reqt, rs.RequestID, data);
}
//Make sure that when we drop the lock, rs is totally initialized
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}",
_info, rs.RequestID, sender);
#endif
try {
rs.Send();
return(rs.RequestID);
}
catch (SendException sx) {
if (sx.IsTransient)
{
//I guess we will just try to resend again in the future:
return(rs.RequestID);
}
else
{
//This is certainly going to fail, so fail now:
StopRequest(rs.RequestID, reply);
throw;
}
}
catch {
//Clean up:
StopRequest(rs.RequestID, reply);
throw;
}
}
protected void HandleRequestAck(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path)
{
RequestState reqs;
lock ( _sync ) {
if (_req_state_table.TryGet(idnum, out reqs))
{
if (reqs.AddAck(ret_path))
{
/*
* Let's look at how long it took to get this reply:
*/
TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate;
_to_mgr.AddAckSampleFor(reqs, ret_path, rtt);
}
}
}
}
protected ICopyable MakeRequest(ReqrepType rt, int next_rep, ICopyable data) {
byte[] header = new byte[ 5 ];
header[0] = (byte)rt;
NumberSerializer.WriteInt( next_rep, header, 1 );
MemBlock mb_header = MemBlock.Reference(header);
return new CopyList(_prefix, mb_header, data);
}
/**
* @param sender how to send the request
* @param reqt the type of request to make
* @param data the data to encapsulate and send
* @param reply the handler to handle the reply
* @param state some state object to attach to this request
* @return the identifier for this request
*
*/
public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data,
IReplyHandler reply, object state)
{
if ( reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest ) {
throw new Exception("Not a request");
}
TimeSpan timeout = _to_mgr.GetTimeOutFor(sender);
RequestState rs = new RequestState(timeout, _to_mgr.AckedTimeOut);
rs.Sender = sender;
rs.ReplyHandler = reply;
rs.RequestType = reqt;
rs.UserState = state;
lock( _sync ) {
rs.RequestID = _req_state_table.GenerateID(rs);
rs.Request = MakeRequest(reqt, rs.RequestID, data);
}
//Make sure that when we drop the lock, rs is totally initialized
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}",
_info, rs.RequestID, sender);
#endif
try {
rs.Send();
return rs.RequestID;
}
catch(SendException sx) {
if( sx.IsTransient ) {
//I guess we will just try to resend again in the future:
return rs.RequestID;
}
else {
//This is certainly going to fail, so fail now:
StopRequest(rs.RequestID, reply);
throw;
}
}
catch {
//Clean up:
StopRequest(rs.RequestID, reply);
throw;
}
}
/**
* When we get a reply ack, we can remove the item from our cache,
* we know the other guy got our reply
*/
protected void HandleReplyAck(ReqrepType rt, int idnum,
MemBlock err_data, ISender ret_path) {
RequestKey rk = new RequestKey(idnum, ret_path);
lock( _sync ) {
/**
* This is not completely safe, but probably fine. Consider the
* case where:
* A -(req)-> B
* A timeout but B does get the req
* A <-(rep)- B
* A -(req)-> B (these cross in flight)
* A -(repack)-> B
*
* but then the repack passes the req retransmission (out of order
* delivery)
*
* This is unlikely, but we could improve it.
* @todo improve the reply caching algorithm
*/
ReplyState rs = (ReplyState)_reply_cache[rk];
if( rs != null ) {
ReleaseReplyState(rs);
}
}
}
protected void HandleError(ReqrepType rt, int idnum,
MemBlock err_data, ISender ret_path)
{
//Get the request:
RequestState reqs;
bool act;
lock( _sync ) {
///@todo, we might not want to stop listening after one error
act = _req_state_table.TryTake(idnum, out reqs);
}
if( act ) {
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepManager: {0}] Receiving error on request id: {1}, from: {2}",
_info, idnum, ret_path);
#endif
///@todo make sure we are checking that this ret_path makes sense for
///our request
ReqrepError rrerr = (ReqrepError)err_data[0];
reqs.ReplyHandler.HandleError(this, idnum, rrerr, ret_path, reqs.UserState);
}
else {
//We have already dealt with this Request
}
}
protected void HandleRequestAck(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) {
RequestState reqs;
lock( _sync ) {
if( _req_state_table.TryGet(idnum, out reqs)) {
if (reqs.AddAck(ret_path)) {
/*
* Let's look at how long it took to get this reply:
*/
TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate;
_to_mgr.AddAckSampleFor(reqs, ret_path, rtt);
}
}
}
}
protected void HandleReply(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) {
RequestState reqs;
if( _req_state_table.TryGet(idnum, out reqs) ) {
IReplyHandler handler = null;
lock( _sync ) {
if (reqs.AddReplier(ret_path)) {
TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate;
_to_mgr.AddReplySampleFor(reqs, ret_path, rtt);
handler = reqs.ReplyHandler;
}
}
/*
* Now handle this reply
*/
if( null != handler ) {
MemBlock payload;
PType pt = PType.Parse(rest, out payload);
Statistics statistics = new Statistics();
statistics.SendCount = reqs.SendCount;
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepManager: {0}] Receiving reply on request id: {1}, from: {2}",
_info, idnum, ret_path);
#endif
//Don't hold the lock while calling the ReplyHandler:
bool continue_listening = handler.HandleReply(this, rt, idnum, pt, payload,
ret_path, statistics, reqs.UserState);
//the request has been served
if( !continue_listening ) {
StopRequest(idnum, handler);
}
}
}
else {
//We are ignoring this reply, it either makes no sense, or we have
//already handled it
}
}
/**
* @param sender how to send the request
* @param reqt the type of request to make
* @param data the data to encapsulate and send
* @param reply the handler to handle the reply
* @param state some state object to attach to this request
* @return the identifier for this request
*
*/
public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data,
IReplyHandler reply, object state)
{
if ( reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest ) {
throw new Exception("Not a request");
}
RequestState rs = new RequestState();
rs.Sender = sender;
rs.ReplyHandler = reply;
rs.RequestType = reqt;
rs.UserState = state;
lock( _sync ) {
//Get the index
int next_req = 0;
do {
next_req = _rand.Next();
} while( _req_state_table.ContainsKey( next_req ) );
/*
* Now we store the request
*/
rs.RequestID = next_req;
rs.Request = MakeRequest(reqt, next_req, data);
_req_state_table[ rs.RequestID ] = rs;
}
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}",
_info, rs.RequestID, sender);
#endif
try {
rs.Send();
return rs.RequestID;
}
catch {
//Clean up:
StopRequest(rs.RequestID, reply);
throw;
}
}
protected void HandleRequest(ReqrepType rt, int idnum,
MemBlock rest, ISender retpath)
{
/**
* Lets see if we have been asked this question before
*/
ReplyState rs = null;
bool resend = false;
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepManager: {0}] Receiving request id: {1}, from: {2}",
_info, idnum, retpath);
#endif
RequestKey rk = new RequestKey(idnum, retpath);
lock( _sync ) {
rs = (ReplyState)_reply_cache[rk];
if( rs == null ) {
rs = GenerateReplyState(_prefix, rk);
}
else {
resend = true;
}
}
if( resend ) {
//This is an old request:
rs.Resend();
}
else {
//This is a new request:
try {
_sub.Handle(rest, rs);
}
catch {
lock( _sync ) {
ReleaseReplyState(rs);
}
//This didn't work out:
try {
MemBlock err_data = MemBlock.Reference(
new byte[]{ (byte) ReqrepError.HandlerFailure } );
ICopyable reply = MakeRequest(ReqrepType.Error, idnum, err_data);
retpath.Send(reply);
}
catch {
//If this fails, we may think about logging.
//The return path could fail, that's the only obvious exception
///@todo log exception
}
}
}
}
protected void HandleError(ReqrepType rt, int idnum,
MemBlock err_data, ISender ret_path)
{
//Get the request:
RequestState reqs = (RequestState)_req_state_table[idnum];
if( reqs != null ) {
bool handle_error = false;
lock( _sync ) {
//Check to see if the request is still good, don't handle
//the error twice:
handle_error = _req_state_table.Contains(idnum);
if( handle_error ) {
///@todo, we might not want to stop listening after one error
_req_state_table.Remove(idnum);
}
}
if( handle_error ) {
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepManager: {0}] Receiving error on request id: {1}, from: {2}",
_info, idnum, ret_path);
#endif
///@todo make sure we are checking that this ret_path makes sense for
///our request
ReqrepError rrerr = (ReqrepError)err_data[0];
reqs.ReplyHandler.HandleError(this, idnum, rrerr, ret_path, reqs.UserState);
}
}
else {
//We have already dealt with this Request
}
}
protected void HandleReply(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) {
RequestState reqs = (RequestState)_req_state_table[idnum];
if( reqs != null ) {
IReplyHandler handler = null;
lock( _sync ) {
if (reqs.AddReplier(ret_path)) {
TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate;
/*
* Let's look at how long it took to get this reply:
*/
if( reqs.GotAck ) {
//Use more standard deviations for acked messages. We
//just don't want to let it run forever.
_acked_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_acked_rtt_stats,
_MINIMUM_TIMEOUT, 3 *_STD_DEVS);
}
else if( ret_path is Edge ) {
_edge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_edge_rtt_stats,
_MINIMUM_TIMEOUT, _STD_DEVS);
}
else {
_nonedge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_nonedge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS);
}
handler = reqs.ReplyHandler;
}
}
/*
* Now handle this reply
*/
if( null != handler ) {
MemBlock payload;
PType pt = PType.Parse(rest, out payload);
Statistics statistics = new Statistics();
statistics.SendCount = reqs.SendCount;
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepManager: {0}] Receiving reply on request id: {1}, from: {2}",
_info, idnum, ret_path);
#endif
//Don't hold the lock while calling the ReplyHandler:
bool continue_listening = handler.HandleReply(this, rt, idnum, pt, payload,
ret_path, statistics, reqs.UserState);
//the request has been served
if( !continue_listening ) {
StopRequest(idnum, handler);
}
}
}
else {
//We are ignoring this reply, it either makes no sense, or we have
//already handled it
}
}
protected void HandleRequestAck(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path) {
RequestState reqs = (RequestState)_req_state_table[idnum];
if( reqs != null ) {
lock( _sync ) {
if (reqs.AddAck(ret_path)) {
/*
* Let's look at how long it took to get this reply:
*/
TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate;
if( ret_path is Edge ) {
_edge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_edge_rtt_stats,
_MINIMUM_TIMEOUT, _STD_DEVS);
}
else {
_nonedge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_nonedge_rtt_stats,
_MINIMUM_TIMEOUT, _STD_DEVS);
}
}
}
}
}
/**
* @param sender how to send the request
* @param reqt the type of request to make
* @param data the data to encapsulate and send
* @param reply the handler to handle the reply
* @param state some state object to attach to this request
* @return the identifier for this request
*
*/
public int SendRequest(ISender sender, ReqrepType reqt, ICopyable data,
IReplyHandler reply, object state)
{
if ( reqt != ReqrepType.Request && reqt != ReqrepType.LossyRequest ) {
throw new Exception("Not a request");
}
TimeSpan timeout = sender is Edge ? _edge_reqtimeout : _nonedge_reqtimeout;
RequestState rs = new RequestState(timeout, _acked_reqtimeout);
rs.Sender = sender;
rs.ReplyHandler = reply;
rs.RequestType = reqt;
rs.UserState = state;
lock( _sync ) {
rs.RequestID = _req_state_table.GenerateID(rs);
rs.Request = MakeRequest(reqt, rs.RequestID, data);
}
//Make sure that when we drop the lock, rs is totally initialized
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepClient: {0}] Sending a request: {1} to node: {2}",
_info, rs.RequestID, sender);
#endif
try {
rs.Send();
return rs.RequestID;
}
catch {
//Clean up:
StopRequest(rs.RequestID, reply);
throw;
}
}
protected void HandleReply(ReqrepType rt, int idnum, MemBlock rest, ISender ret_path)
{
RequestState reqs;
if (_req_state_table.TryGet(idnum, out reqs))
{
IReplyHandler handler = null;
lock ( _sync ) {
if (reqs.AddReplier(ret_path))
{
TimeSpan rtt = DateTime.UtcNow - reqs.ReqDate;
/*
* Let's look at how long it took to get this reply:
*/
if (reqs.GotAck)
{
//Use more standard deviations for acked messages. We
//just don't want to let it run forever.
_acked_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_acked_rtt_stats,
_MINIMUM_TIMEOUT, 3 * _STD_DEVS);
}
else if (ret_path is Edge)
{
_edge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_edge_rtt_stats,
_MINIMUM_TIMEOUT, _STD_DEVS);
}
else
{
_nonedge_reqtimeout = ComputeNewTimeOut(rtt.TotalMilliseconds,
_nonedge_rtt_stats, _MINIMUM_TIMEOUT, _STD_DEVS);
}
handler = reqs.ReplyHandler;
}
}
/*
* Now handle this reply
*/
if (null != handler)
{
MemBlock payload;
PType pt = PType.Parse(rest, out payload);
Statistics statistics = new Statistics();
statistics.SendCount = reqs.SendCount;
#if REQREP_DEBUG
Console.Error.WriteLine("[ReqrepManager: {0}] Receiving reply on request id: {1}, from: {2}",
_info, idnum, ret_path);
#endif
//Don't hold the lock while calling the ReplyHandler:
bool continue_listening = handler.HandleReply(this, rt, idnum, pt, payload,
ret_path, statistics, reqs.UserState);
//the request has been served
if (!continue_listening)
{
StopRequest(idnum, handler);
}
}
}
else
{
//We are ignoring this reply, it either makes no sense, or we have
//already handled it
}
}
