public void SendBuffer_must_throw_exception_if_nonbuffered_sequence_number_is_NACKed()
{
var b0 = new AckedSendBuffer <Sequenced>(10);
var msg1 = Msg(1);
var msg2 = Msg(2);
var b1 = b0.Buffer(msg1).Buffer(msg2);
XAssert.Throws <ResendUnfulfillableException>(() => b1.Acknowledge(new Ack(new SeqNo(2), new [] { new SeqNo(0) })));
}
public void SendBuffer_must_remove_messages_from_buffer_when_cumulative_ack_received()
{
var b0 = new AckedSendBuffer <Sequenced>(10);
var msg0 = Msg(0);
var msg1 = Msg(1);
var msg2 = Msg(2);
var msg3 = Msg(3);
var msg4 = Msg(4);
var b1 = b0.Buffer(msg0);
Assert.True(b1.NonAcked.SequenceEqual(new[] { msg0 }));
var b2 = b1.Buffer(msg1);
Assert.True(b2.NonAcked.SequenceEqual(new[] { msg0, msg1 }));
var b3 = b2.Buffer(msg2);
Assert.True(b3.NonAcked.SequenceEqual(new[] { msg0, msg1, msg2 }));
var b4 = b3.Acknowledge(new Ack(new SeqNo(1)));
Assert.True(b4.NonAcked.SequenceEqual(new[] { msg2 }));
var b5 = b4.Buffer(msg3);
Assert.True(b5.NonAcked.SequenceEqual(new [] { msg2, msg3 }));
var b6 = b5.Buffer(msg4);
Assert.True(b6.NonAcked.SequenceEqual(new[] { msg2, msg3, msg4 }));
var b7 = b6.Acknowledge(new Ack(new SeqNo(1)));
Assert.True(b7.NonAcked.SequenceEqual(new[] { msg2, msg3, msg4 }));
var b8 = b7.Acknowledge(new Ack(new SeqNo(2)));
Assert.True(b8.NonAcked.SequenceEqual(new[] { msg3, msg4 }));
var b9 = b8.Acknowledge(new Ack(new SeqNo(4)));
Assert.True(b9.NonAcked.Count == 0);
}
public void SendBuffer_must_aggregate_unacked_messages_in_order()
{
var b0 = new AckedSendBuffer <Sequenced>(10);
var msg0 = Msg(0);
var msg1 = Msg(1);
var msg2 = Msg(2);
var b1 = b0.Buffer(msg0);
Assert.True(b1.NonAcked.SequenceEqual(new[] { msg0 }));
var b2 = b1.Buffer(msg1);
Assert.True(b2.NonAcked.SequenceEqual(new[] { msg0, msg1 }));
var b3 = b2.Buffer(msg2);
Assert.True(b3.NonAcked.SequenceEqual(new[] { msg0, msg1, msg2 }));
}
public void SendBuffer_must_keep_NACKed_messages_in_buffer_if_selective_nacks_are_received()
{
var b0 = new AckedSendBuffer <Sequenced>(10);
var msg0 = Msg(0);
var msg1 = Msg(1);
var msg2 = Msg(2);
var msg3 = Msg(3);
var msg4 = Msg(4);
var b1 = b0.Buffer(msg0);
Assert.True(b1.NonAcked.SequenceEqual(new[] { msg0 }));
var b2 = b1.Buffer(msg1);
Assert.True(b2.NonAcked.SequenceEqual(new[] { msg0, msg1 }));
var b3 = b2.Buffer(msg2);
Assert.True(b3.NonAcked.SequenceEqual(new[] { msg0, msg1, msg2 }));
var b4 = b3.Acknowledge(new Ack(new SeqNo(1), new [] { new SeqNo(0) }));
Assert.True(b4.NonAcked.SequenceEqual(new [] { msg2 }));
Assert.True(b4.Nacked.SequenceEqual(new[] { msg0 }));
var b5 = b4.Buffer(msg3).Buffer(msg4);
Assert.True(b5.NonAcked.SequenceEqual(new[] { msg2, msg3, msg4 }));
Assert.True(b5.Nacked.SequenceEqual(new[] { msg0 }));
var b6 = b5.Acknowledge(new Ack(new SeqNo(4), new [] { new SeqNo(2), new SeqNo(3) }));
Assert.True(b6.NonAcked.Count == 0);
Assert.True(b6.Nacked.SequenceEqual(new[] { msg2, msg3 }));
var b7 = b6.Acknowledge(new Ack(new SeqNo(4)));
Assert.True(b7.NonAcked.Count == 0);
Assert.True(b7.Nacked.Count == 0);
}
public void SendBuffer_and_ReceiveBuffer_must_correctly_cooperate_with_each_other()
{
var msgCount = 1000;
var deliveryProbability = 0.5D;
var referenceList = Enumerable.Range(0, msgCount).Select(x => Msg(x)).ToList();
var toSend = referenceList;
var received = new List <Sequenced>();
var sndBuf = new AckedSendBuffer <Sequenced>(10);
var rcvBuf = new AckedReceiveBuffer <Sequenced>();
var log = new List <string>();
var lastAck = new Ack(new SeqNo(-1));
Action <string> dbLog = log.Add;
Action <int, double> senderSteps = (steps, p) =>
{
var resends = new List <Sequenced>(sndBuf.Nacked.Concat(sndBuf.NonAcked).Take(steps));
var sends = new List <Sequenced>();
if (steps - resends.Count > 0)
{
var tmp = toSend.Take(steps - resends.Count).ToList();
toSend = toSend.Drop(steps - resends.Count).ToList();
sends = tmp;
}
foreach (var msg in resends.Concat(sends))
{
if (sends.Contains(msg))
{
sndBuf = sndBuf.Buffer(msg);
}
if (Happened(p))
{
var del = rcvBuf.Receive(msg).ExtractDeliverable();
rcvBuf = del.Buffer;
dbLog(string.Format("{0} -- {1} --> {2}", sndBuf, msg, rcvBuf));
lastAck = del.Ack;
received.AddRange(del.Deliverables);
dbLog(string.Format("R: {0}", string.Join(",", received.Select(x => x.ToString()))));
}
else
{
dbLog(string.Format("{0} -- {1} --X {2}", sndBuf, msg, rcvBuf));
}
}
};
Action <double> receiverStep = (p) =>
{
if (Happened(p))
{
sndBuf = sndBuf.Acknowledge(lastAck);
dbLog(string.Format("{0} <-- {1} -- {2}", sndBuf, lastAck, rcvBuf));
}
else
{
dbLog(string.Format("{0} X-- {1} -- {2}", sndBuf, lastAck, rcvBuf));
}
};
//Dropping phase
global::System.Diagnostics.Debug.WriteLine("Starting unreliable delivery for {0} messages, with delivery probably P = {1}", msgCount, deliveryProbability);
var nextSteps = msgCount * 2;
while (nextSteps > 0)
{
var s = Geom(0.3, limit: 5);
senderSteps(s, deliveryProbability);
receiverStep(deliveryProbability);
nextSteps--;
}
global::System.Diagnostics.Debug.WriteLine("Successfully delivered {0} messages from {1}", received.Count, msgCount);
global::System.Diagnostics.Debug.WriteLine("Entering reliable phase");
//Finalizing phase
for (var i = 1; i <= msgCount; i++)
{
senderSteps(1, 1.0);
receiverStep(1.0);
}
if (!received.SequenceEqual(referenceList))
{
global::System.Diagnostics.Debug.WriteLine(string.Join(Environment.NewLine, log));
global::System.Diagnostics.Debug.WriteLine("Received: ");
global::System.Diagnostics.Debug.WriteLine(string.Join(Environment.NewLine, received.Select(x => x.ToString())));
Assert.True(false, "Not all messages were received");
}
global::System.Diagnostics.Debug.WriteLine("All messages have been successfully delivered");
}
public void SendBuffer_must_refuse_buffering_new_messages_if_capacity_reached()
{
var buffer = new AckedSendBuffer <Sequenced>(4).Buffer(Msg(0)).Buffer(Msg(1)).Buffer(Msg(2)).Buffer(Msg(3));
XAssert.Throws <ResendBufferCapacityReachedException>(() => buffer.Buffer(Msg(4)));
}
protected void Receiving()
{
Receive<EndpointWriter.FlushAndStop>(flush =>
{
//Trying to serve until our last breath
ResendAll();
_writer.Tell(EndpointWriter.FlushAndStop.Instance);
Become(FlushWait);
});
Receive<IsIdle>(idle => { }); // Do not reply, we will Terminate soon, or send a GotUid
Receive<EndpointManager.Send>(send => HandleSend(send));
Receive<Ack>(ack =>
{
// If we are not sure about the UID just ignore the ack. Ignoring is fine.
if (UidConfirmed)
{
try
{
_resendBuffer = _resendBuffer.Acknowledge(ack);
}
catch (Exception ex)
{
throw new HopelessAssociation(_localAddress, _remoteAddress, Uid,
new IllegalStateException($"Error encountered while processing system message acknowledgement buffer: {_resendBuffer} ack: {ack}", ex));
}
ResendNacked();
}
});
Receive<AttemptSysMsgRedelivery>(sysmsg =>
{
if (UidConfirmed) ResendAll();
});
Receive<Terminated>(terminated =>
{
_currentHandle = null;
Context.Parent.Tell(new EndpointWriter.StoppedReading(Self));
if (_resendBuffer.NonAcked.Any() || _resendBuffer.Nacked.Any())
Context.System.Scheduler.ScheduleTellOnce(_settings.SysResendTimeout, Self,
new AttemptSysMsgRedelivery(), Self);
GoToIdle();
});
Receive<GotUid>(g =>
{
_bailoutAt = null;
Context.Parent.Tell(g);
//New system that has the same address as the old - need to start from fresh state
UidConfirmed = true;
if (Uid.HasValue && Uid.Value != g.Uid) Reset();
Uid = g.Uid;
ResendAll();
});
Receive<EndpointWriter.StopReading>(stopped =>
{
_writer.Forward(stopped); //forward the request
});
}
protected void FlushWait(object message)
{
if (message is Terminated)
{
//Clear buffer to prevent sending system messages to dead letters -- at this point we are shutting down and
//don't know if they were properly delivered or not
_resendBuffer = new AckedSendBuffer<EndpointManager.Send>(0);
Context.Stop(Self);
}
}
public void SendBuffer_must_throw_exception_if_nonbuffered_sequence_number_is_NACKed()
{
var b0 = new AckedSendBuffer<Sequenced>(10);
var msg1 = Msg(1);
var msg2 = Msg(2);
var b1 = b0.Buffer(msg1).Buffer(msg2);
XAssert.Throws<ResendUnfulfillableException>(() => b1.Acknowledge(new Ack(new SeqNo(2), new []{ new SeqNo(0) })));
}
public void SendBuffer_must_keep_NACKed_messages_in_buffer_if_selective_nacks_are_received()
{
var b0 = new AckedSendBuffer<Sequenced>(10);
var msg0 = Msg(0);
var msg1 = Msg(1);
var msg2 = Msg(2);
var msg3 = Msg(3);
var msg4 = Msg(4);
var b1 = b0.Buffer(msg0);
Assert.True(b1.NonAcked.SequenceEqual(new[] { msg0 }));
var b2 = b1.Buffer(msg1);
Assert.True(b2.NonAcked.SequenceEqual(new[] { msg0, msg1 }));
var b3 = b2.Buffer(msg2);
Assert.True(b3.NonAcked.SequenceEqual(new[] { msg0, msg1, msg2 }));
var b4 = b3.Acknowledge(new Ack(new SeqNo(1), new [] {new SeqNo(0)}));
Assert.True(b4.NonAcked.SequenceEqual(new []{ msg2 }));
Assert.True(b4.Nacked.SequenceEqual(new[] { msg0 }));
var b5 = b4.Buffer(msg3).Buffer(msg4);
Assert.True(b5.NonAcked.SequenceEqual(new[] { msg2, msg3, msg4 }));
Assert.True(b5.Nacked.SequenceEqual(new[] { msg0 }));
var b6 = b5.Acknowledge(new Ack(new SeqNo(4), new []{new SeqNo(2), new SeqNo(3)}));
Assert.True(b6.NonAcked.Count == 0);
Assert.True(b6.Nacked.SequenceEqual(new[] { msg2, msg3 }));
var b7 = b6.Acknowledge(new Ack(new SeqNo(5)));
Assert.True(b7.NonAcked.Count == 0);
Assert.True(b7.Nacked.Count == 0);
}
public void SendBuffer_must_remove_messages_from_buffer_when_cumulative_ack_received()
{
var b0 = new AckedSendBuffer<Sequenced>(10);
var msg0 = Msg(0);
var msg1 = Msg(1);
var msg2 = Msg(2);
var msg3 = Msg(3);
var msg4 = Msg(4);
var b1 = b0.Buffer(msg0);
Assert.True(b1.NonAcked.SequenceEqual(new[] { msg0 }));
var b2 = b1.Buffer(msg1);
Assert.True(b2.NonAcked.SequenceEqual(new[] { msg0, msg1 }));
var b3 = b2.Buffer(msg2);
Assert.True(b3.NonAcked.SequenceEqual(new[] { msg0, msg1, msg2 }));
var b4 = b3.Acknowledge(new Ack(new SeqNo(1)));
Assert.True(b4.NonAcked.SequenceEqual(new[]{ msg2 }));
var b5 = b4.Buffer(msg3);
Assert.True(b5.NonAcked.SequenceEqual(new []{ msg2,msg3 }));
var b6 = b5.Buffer(msg4);
Assert.True(b6.NonAcked.SequenceEqual(new[] { msg2, msg3, msg4 }));
var b7 = b6.Acknowledge(new Ack(new SeqNo(1)));
Assert.True(b7.NonAcked.SequenceEqual(new[] { msg2, msg3, msg4 }));
var b8 = b7.Acknowledge(new Ack(new SeqNo(2)));
Assert.True(b8.NonAcked.SequenceEqual(new[] { msg3, msg4 }));
var b9 = b8.Acknowledge(new Ack(new SeqNo(5)));
Assert.True(b9.NonAcked.Count == 0);
}
public void SendBuffer_must_refuse_buffering_new_messages_if_capacity_reached()
{
var buffer = new AckedSendBuffer<Sequenced>(4).Buffer(Msg(0)).Buffer(Msg(1)).Buffer(Msg(2)).Buffer(Msg(3));
XAssert.Throws<ResendBufferCapacityReachedException>(() => buffer.Buffer(Msg(4)));
}
public void SendBuffer_must_aggregate_unacked_messages_in_order()
{
var b0 = new AckedSendBuffer<Sequenced>(10);
var msg0 = Msg(0);
var msg1 = Msg(1);
var msg2 = Msg(2);
var b1 = b0.Buffer(msg0);
Assert.True(b1.NonAcked.SequenceEqual(new[]{msg0}));
var b2 = b1.Buffer(msg1);
Assert.True(b2.NonAcked.SequenceEqual(new[] { msg0, msg1 }));
var b3 = b2.Buffer(msg2);
Assert.True(b3.NonAcked.SequenceEqual(new[] { msg0, msg1, msg2 }));
}
protected void FlushWait()
{
Receive<IsIdle>(idle => { }); // Do not reply, we will Terminate soon, which will do the inbound connection unstashing
Receive<Terminated>(terminated =>
{
//Clear buffer to prevent sending system messages to dead letters -- at this point we are shutting down and
//don't know if they were properly delivered or not
_resendBuffer = new AckedSendBuffer<EndpointManager.Send>(0);
Context.Stop(Self);
});
ReceiveAny(o => { }); // ignore
}
private void Reset()
{
_resendBuffer = new AckedSendBuffer<EndpointManager.Send>(_settings.SysMsgBufferSize);
ScheduleAutoResend();
_lastCumulativeAck = new SeqNo(-1);
_seqCounter = 0L;
_pendingAcks = new List<Ack>();
}
protected override void OnReceive(object message)
{
message.Match()
.With<EndpointWriter.FlushAndStop>(flush =>
{
//Trying to serve untilour last breath
ResendAll();
_writer.Tell(EndpointWriter.FlushAndStop.Instance);
Context.Become(FlushWait);
})
.With<EndpointManager.Send>(HandleSend)
.With<Ack>(ack =>
{
if (!UidConfirmed) _pendingAcks.Add(ack);
else
{
try
{
_resendBuffer = _resendBuffer.Acknowledge(ack);
}
catch (Exception ex)
{
throw new InvalidAssociationException(
string.Format(
"Error encountered while processing system message acknowledgement {0} {1}",
_resendBuffer, ack), ex);
}
if (_lastCumulativeAck < ack.CumulativeAck)
{
_lastCumulativeAck = ack.CumulativeAck;
// Cumulative ack is progressing, we might not need to resend non-acked messages yet.
// If this progression stops, the timer will eventually kick in, since scheduleAutoResend
// does not cancel existing timers (see the "else" case).
RescheduleAutoResend();
}
else
{
ScheduleAutoResend();
}
ResendNacked();
}
})
.With<AttemptSysMsgRedelivery>(sysmsg =>
{
if (UidConfirmed) ResendAll();
})
.With<Terminated>(terminated =>
{
_currentHandle = null;
Context.Parent.Tell(new EndpointWriter.StoppedReading(Self));
if (_resendBuffer.NonAcked.Count > 0 || _resendBuffer.Nacked.Count > 0)
Context.System.Scheduler.ScheduleTellOnce(_settings.SysResendTimeout, Self,
new AttemptSysMsgRedelivery(), Self);
Context.Become(Idle);
})
.With<GotUid>(g =>
{
Context.Parent.Tell(g);
//New system that has the same address as the old - need to start from fresh state
UidConfirmed = true;
if (Uid.HasValue && Uid.Value != g.Uid) Reset();
else UnstashAcks();
Uid = _refuseUid;
})
.With<EndpointWriter.StopReading>(stopped =>
{
_writer.Forward(stopped); //forward the request
});
}
public void SendBuffer_and_ReceiveBuffer_must_correctly_cooperate_with_each_other()
{
var msgCount = 1000;
var deliveryProbability = 0.5D;
var referenceList = Enumerable.Range(0, msgCount).Select(x => Msg(x)).ToList();
var toSend = referenceList;
var received = new List<Sequenced>();
var sndBuf = new AckedSendBuffer<Sequenced>(10);
var rcvBuf = new AckedReceiveBuffer<Sequenced>();
var log = new List<string>();
var lastAck = new Ack(new SeqNo(-1));
Action<string> dbLog = log.Add;
Action<int, double> senderSteps = (steps, p) =>
{
var resends = new List<Sequenced>(sndBuf.Nacked.Concat(sndBuf.NonAcked).Take(steps));
var sends = new List<Sequenced>();
if (steps - resends.Count > 0)
{
var tmp = toSend.Take(steps - resends.Count).ToList();
toSend = toSend.Drop(steps - resends.Count).ToList();
sends = tmp;
}
foreach (var msg in resends.Concat(sends))
{
if (sends.Contains(msg)) sndBuf = sndBuf.Buffer(msg);
if (Happened(p))
{
var del = rcvBuf.Receive(msg).ExtractDeliverable;
rcvBuf = del.Buffer;
dbLog(string.Format("{0} -- {1} --> {2}", sndBuf, msg, rcvBuf));
lastAck = del.Ack;
received.AddRange(del.Deliverables);
dbLog(string.Format("R: {0}", string.Join(",", received.Select(x => x.ToString()))));
}
else
{
dbLog(string.Format("{0} -- {1} --X {2}", sndBuf, msg, rcvBuf));
}
}
};
Action<double> receiverStep = (p) =>
{
if (Happened(p))
{
sndBuf = sndBuf.Acknowledge(lastAck);
dbLog(string.Format("{0} <-- {1} -- {2}", sndBuf, lastAck, rcvBuf));
}
else
{
dbLog(string.Format("{0} X-- {1} -- {2}", sndBuf, lastAck, rcvBuf));
}
};
//Dropping phase
global::System.Diagnostics.Debug.WriteLine("Starting unreliable delivery for {0} messages, with delivery probably P = {1}", msgCount, deliveryProbability);
var nextSteps = msgCount*2;
while (nextSteps > 0)
{
var s = Geom(0.3, limit: 5);
senderSteps(s, deliveryProbability);
receiverStep(deliveryProbability);
nextSteps--;
}
global::System.Diagnostics.Debug.WriteLine("Successfully delivered {0} messages from {1}", received.Count, msgCount);
global::System.Diagnostics.Debug.WriteLine("Entering reliable phase");
//Finalizing phase
for (var i = 1; i <= msgCount; i++)
{
senderSteps(1, 1.0);
receiverStep(1.0);
}
if (!received.SequenceEqual(referenceList))
{
global::System.Diagnostics.Debug.WriteLine(string.Join(Environment.NewLine, log));
global::System.Diagnostics.Debug.WriteLine("Received: ");
global::System.Diagnostics.Debug.WriteLine(string.Join(Environment.NewLine, received.Select(x => x.ToString())));
Assert.True(false,"Not all messages were received");
}
global::System.Diagnostics.Debug.WriteLine("All messages have been successfully delivered");
}
private void TryBuffer(EndpointManager.Send s)
{
try
{
_resendBuffer = _resendBuffer.Buffer(s);
}
catch (Exception ex)
{
throw new HopelessAssociation(_localAddress, _remoteAddress, Uid, ex);
}
}
private void Reset()
{
_resendBuffer = new AckedSendBuffer<EndpointManager.Send>(_settings.SysMsgBufferSize);
_seqCounter = 0L;
_bailoutAt = null;
_bufferWasInUse = false;
}