public void ManyMessagesShouldNotTriggerSizeAlert()
{
int messageSize = TestUtil.GetAllSet().SerializedSize;
// Enough messages to trigger the alert unless we've reset the size
// Note that currently we need to make this big enough to copy two whole buffers,
// as otherwise when we refill the buffer the second type, the alert triggers instantly.
int correctCount = (CodedInputStream.BufferSize * 2) / messageSize + 1;
using (MemoryStream stream = new MemoryStream()) {
MessageStreamWriter<TestAllTypes> writer = new MessageStreamWriter<TestAllTypes>(stream);
for (int i = 0; i < correctCount; i++) {
writer.Write(TestUtil.GetAllSet());
}
writer.Flush();
stream.Position = 0;
int count = 0;
foreach (var message in MessageStreamIterator<TestAllTypes>.FromStreamProvider(() => stream)
.WithSizeLimit(CodedInputStream.BufferSize * 2)) {
count++;
TestUtil.AssertAllFieldsSet(message);
}
Assert.AreEqual(correctCount, count);
}
}
private void PersistTopology()
{
byte[] buffer;
var topology = master.Topology.GetTopology();
using (var stream = new MemoryStream())
{
var writer = new MessageStreamWriter <TopologyResultMessage>(stream);
writer.Write(topology);
writer.Flush();
buffer = stream.ToArray();
}
hashTable.Batch(actions =>
{
var values = actions.Get(new GetRequest
{
Key = Constants.Topology
});
actions.Put(new PutRequest
{
Key = Constants.Topology,
ParentVersions = values.Select(x => x.Version).ToArray(),
Bytes = buffer,
IsReadOnly = true
});
actions.Commit();
});
}
public PutResult[] Put(int topologyVersion,
params ExtendedPutRequest[] valuesToAdd)
{
writer.Write(new StorageMessageUnion.Builder
{
Type = StorageMessageType.PutRequests,
TopologyVersion = topologyVersion,
PutRequestsList =
{
valuesToAdd.Select(x => x.GetPutRequest())
}
}.Build());
writer.Flush();
stream.Flush();
var union = ReadReply(StorageMessageType.PutResponses);
return(union.PutResponsesList.Select(x => new PutResult
{
ConflictExists = x.ConflictExists,
Version = new PersistentHashTable.ValueVersion
{
InstanceId = new Guid(x.Version.InstanceId.ToByteArray()),
Number = x.Version.Number
}
}).ToArray());
}
public void ThreeMessages() {
NestedMessage message1 = new NestedMessage.Builder { Bb = 5 }.Build();
NestedMessage message2 = new NestedMessage.Builder { Bb = 1500 }.Build();
NestedMessage message3 = new NestedMessage.Builder().Build();
byte[] data;
using (MemoryStream stream = new MemoryStream()) {
MessageStreamWriter<NestedMessage> writer = new MessageStreamWriter<NestedMessage>(stream);
writer.Write(message1);
writer.Write(message2);
writer.Write(message3);
writer.Flush();
data = stream.ToArray();
}
TestUtil.AssertEqualBytes(ThreeMessageData, data);
}
private void OnBeginAcceptTcpClient(IAsyncResult result)
{
TcpClient client;
try
{
client = listener.EndAcceptTcpClient(result);
listener.BeginAcceptTcpClient(OnBeginAcceptTcpClient, null);
}
catch (ObjectDisposedException)
{
return;
}
catch (InvalidOperationException)
{
return;
}
try
{
using (client)
using (var stream = client.GetStream())
{
var writer = new MessageStreamWriter<StorageMessageUnion>(stream);
try
{
foreach (var wrapper in MessageStreamIterator<StorageMessageUnion>.FromStreamProvider(() => new UndisposableStream(stream)))
{
log.DebugFormat("Got message {0}", wrapper.Type);
switch (wrapper.Type)
{
case StorageMessageType.GetRequests:
HandleGet(wrapper, wrapper.TopologyVersion.Value, writer);
break;
case StorageMessageType.PutRequests:
HandlePut(wrapper, wrapper.TopologyVersion.Value, writer);
break;
case StorageMessageType.RemoveRequests:
HandleRemove(wrapper, wrapper.TopologyVersion.Value, writer);
break;
case StorageMessageType.AssignAllEmptySegmentsRequest:
HandleAssignEmpty(wrapper, writer);
break;
case StorageMessageType.ReplicateNextPageRequest:
HandleReplicateNextPage(wrapper, writer);
break;
case StorageMessageType.UpdateTopology:
HandleTopologyUpdate(writer);
break;
default:
throw new InvalidOperationException("Message type was not understood: " + wrapper.Type);
}
writer.Flush();
stream.Flush();
}
}
catch(IOException)
{
// disconnected, so nothing else to do
}
catch (SeeOtherException e)
{
writer.Write(new StorageMessageUnion.Builder
{
Type = StorageMessageType.SeeOtherError,
SeeOtherError = new SeeOtherErrorMessage.Builder
{
Other = e.Endpoint.GetNodeEndpoint()
}.Build()
}.Build());
writer.Flush();
stream.Flush();
}
catch (TopologyVersionDoesNotMatchException)
{
writer.Write(new StorageMessageUnion.Builder
{
Type = StorageMessageType.TopologyChangedError,
}.Build());
writer.Flush();
stream.Flush();
}
catch (Exception e)
{
log.Warn("Error performing request", e);
writer.Write(new StorageMessageUnion.Builder
{
Type = StorageMessageType.StorageErrorResult,
Exception = new ErrorMessage.Builder
{
Message = e.ToString()
}.Build()
}.Build());
writer.Flush();
stream.Flush();
}
}
}
catch (Exception e)
{
log.Warn("Error when processing request to storage", e);
}
}
private void PersistTopology()
{
byte[] buffer;
var topology = master.Topology.GetTopology();
using (var stream = new MemoryStream())
{
var writer = new MessageStreamWriter<TopologyResultMessage>(stream);
writer.Write(topology);
writer.Flush();
buffer = stream.ToArray();
}
hashTable.Batch(actions =>
{
var values = actions.Get(new GetRequest
{
Key = Constants.Topology
});
actions.Put(new PutRequest
{
Key = Constants.Topology,
ParentVersions = values.Select(x => x.Version).ToArray(),
Bytes = buffer,
IsReadOnly = true
});
actions.Commit();
});
}
private void OnAcceptTcpClient(IAsyncResult result)
{
TcpClient client;
try
{
client = listener.EndAcceptTcpClient(result);
}
catch (ObjectDisposedException)
{
return;
}
//this is done intentionally in a single threaded fashion
//the master is not a hot spot and it drastically simplify our life
//to avoid having to do multi threaded stuff here
//all calls to the master are also very short
try
{
using (client)
using (var stream = client.GetStream())
{
var writer = new MessageStreamWriter<MasterMessageUnion>(stream);
foreach (var wrapper in MessageStreamIterator<MasterMessageUnion>.FromStreamProvider(() => stream))
{
try
{
log.DebugFormat("Accepting message from {0} - {1}",
client.Client.RemoteEndPoint,
wrapper.Type);
switch (wrapper.Type)
{
case MasterMessageType.GetTopologyRequest:
HandleGetToplogy(writer);
break;
case MasterMessageType.JoinRequest:
HandleJoin(wrapper, writer);
break;
case MasterMessageType.CaughtUpRequest:
HandleCatchUp(wrapper, writer);
break;
case MasterMessageType.GaveUpRequest:
HandleGaveUp(wrapper, writer);
break;
default:
throw new ArgumentOutOfRangeException();
}
writer.Flush();
stream.Flush();
}
catch (Exception e)
{
log.Warn("Error performing request", e);
writer.Write(new MasterMessageUnion.Builder
{
Type = MasterMessageType.MasterErrorResult,
Exception = new ErrorMessage.Builder
{
Message = e.ToString()
}.Build()
}.Build());
writer.Flush();
stream.Flush();
}
}
}
}
catch (Exception e)
{
log.Warn("Error when dealing with a request (or could not send error details)", e);
}
finally
{
try
{
listener.BeginAcceptTcpClient(OnAcceptTcpClient, null);
}
catch (InvalidOperationException)
{
//the listener was closed
}
}
}
private void OnAcceptTcpClient(IAsyncResult result)
{
TcpClient client;
try
{
client = listener.EndAcceptTcpClient(result);
}
catch (ObjectDisposedException)
{
return;
}
//this is done intentionally in a single threaded fashion
//the master is not a hot spot and it drastically simplify our life
//to avoid having to do multi threaded stuff here
//all calls to the master are also very short
try
{
using (client)
using (var stream = client.GetStream())
{
var writer = new MessageStreamWriter <MasterMessageUnion>(stream);
foreach (var wrapper in MessageStreamIterator <MasterMessageUnion> .FromStreamProvider(() => stream))
{
try
{
log.DebugFormat("Accepting message from {0} - {1}",
client.Client.RemoteEndPoint,
wrapper.Type);
switch (wrapper.Type)
{
case MasterMessageType.GetTopologyRequest:
HandleGetToplogy(writer);
break;
case MasterMessageType.JoinRequest:
HandleJoin(wrapper, writer);
break;
case MasterMessageType.CaughtUpRequest:
HandleCatchUp(wrapper, writer);
break;
case MasterMessageType.GaveUpRequest:
HandleGaveUp(wrapper, writer);
break;
default:
throw new ArgumentOutOfRangeException();
}
writer.Flush();
stream.Flush();
}
catch (Exception e)
{
log.Warn("Error performing request", e);
writer.Write(new MasterMessageUnion.Builder
{
Type = MasterMessageType.MasterErrorResult,
Exception = new ErrorMessage.Builder
{
Message = e.ToString()
}.Build()
}.Build());
writer.Flush();
stream.Flush();
}
}
}
}
catch (Exception e)
{
log.Warn("Error when dealing with a request (or could not send error details)", e);
}
finally
{
try
{
listener.BeginAcceptTcpClient(OnAcceptTcpClient, null);
}
catch (InvalidOperationException)
{
//the listener was closed
}
}
}
private void OnBeginAcceptTcpClient(IAsyncResult result)
{
TcpClient client;
try
{
client = listener.EndAcceptTcpClient(result);
listener.BeginAcceptTcpClient(OnBeginAcceptTcpClient, null);
}
catch (ObjectDisposedException)
{
return;
}
catch (InvalidOperationException)
{
return;
}
try
{
using (client)
using (var stream = client.GetStream())
{
var writer = new MessageStreamWriter <StorageMessageUnion>(stream);
try
{
foreach (var wrapper in MessageStreamIterator <StorageMessageUnion> .FromStreamProvider(() => new UndisposableStream(stream)))
{
log.DebugFormat("Got message {0}", wrapper.Type);
switch (wrapper.Type)
{
case StorageMessageType.GetRequests:
HandleGet(wrapper, wrapper.TopologyVersion.Value, writer);
break;
case StorageMessageType.PutRequests:
HandlePut(wrapper, wrapper.TopologyVersion.Value, writer);
break;
case StorageMessageType.RemoveRequests:
HandleRemove(wrapper, wrapper.TopologyVersion.Value, writer);
break;
case StorageMessageType.AssignAllEmptySegmentsRequest:
HandleAssignEmpty(wrapper, writer);
break;
case StorageMessageType.ReplicateNextPageRequest:
HandleReplicateNextPage(wrapper, writer);
break;
case StorageMessageType.UpdateTopology:
HandleTopologyUpdate(writer);
break;
default:
throw new InvalidOperationException("Message type was not understood: " + wrapper.Type);
}
writer.Flush();
stream.Flush();
}
}
catch (IOException)
{
// disconnected, so nothing else to do
}
catch (SeeOtherException e)
{
writer.Write(new StorageMessageUnion.Builder
{
Type = StorageMessageType.SeeOtherError,
SeeOtherError = new SeeOtherErrorMessage.Builder
{
Other = e.Endpoint.GetNodeEndpoint()
}.Build()
}.Build());
writer.Flush();
stream.Flush();
}
catch (TopologyVersionDoesNotMatchException)
{
writer.Write(new StorageMessageUnion.Builder
{
Type = StorageMessageType.TopologyChangedError,
}.Build());
writer.Flush();
stream.Flush();
}
catch (Exception e)
{
log.Warn("Error performing request", e);
writer.Write(new StorageMessageUnion.Builder
{
Type = StorageMessageType.StorageErrorResult,
Exception = new ErrorMessage.Builder
{
Message = e.ToString()
}.Build()
}.Build());
writer.Flush();
stream.Flush();
}
}
}
catch (Exception e)
{
log.Warn("Error when processing request to storage", e);
}
}
