public void ManyMessagesShouldNotTriggerSizeAlert() { int messageSize = TestUtil.GetAllSet().SerializedSize; // Enough messages to trigger the alert unless we've reset the size // Note that currently we need to make this big enough to copy two whole buffers, // as otherwise when we refill the buffer the second type, the alert triggers instantly. int correctCount = (CodedInputStream.BufferSize * 2) / messageSize + 1; using (MemoryStream stream = new MemoryStream()) { MessageStreamWriter<TestAllTypes> writer = new MessageStreamWriter<TestAllTypes>(stream); for (int i = 0; i < correctCount; i++) { writer.Write(TestUtil.GetAllSet()); } writer.Flush(); stream.Position = 0; int count = 0; foreach (var message in MessageStreamIterator<TestAllTypes>.FromStreamProvider(() => stream) .WithSizeLimit(CodedInputStream.BufferSize * 2)) { count++; TestUtil.AssertAllFieldsSet(message); } Assert.AreEqual(correctCount, count); } }
private void PersistTopology() { byte[] buffer; var topology = master.Topology.GetTopology(); using (var stream = new MemoryStream()) { var writer = new MessageStreamWriter <TopologyResultMessage>(stream); writer.Write(topology); writer.Flush(); buffer = stream.ToArray(); } hashTable.Batch(actions => { var values = actions.Get(new GetRequest { Key = Constants.Topology }); actions.Put(new PutRequest { Key = Constants.Topology, ParentVersions = values.Select(x => x.Version).ToArray(), Bytes = buffer, IsReadOnly = true }); actions.Commit(); }); }
public PutResult[] Put(int topologyVersion, params ExtendedPutRequest[] valuesToAdd) { writer.Write(new StorageMessageUnion.Builder { Type = StorageMessageType.PutRequests, TopologyVersion = topologyVersion, PutRequestsList = { valuesToAdd.Select(x => x.GetPutRequest()) } }.Build()); writer.Flush(); stream.Flush(); var union = ReadReply(StorageMessageType.PutResponses); return(union.PutResponsesList.Select(x => new PutResult { ConflictExists = x.ConflictExists, Version = new PersistentHashTable.ValueVersion { InstanceId = new Guid(x.Version.InstanceId.ToByteArray()), Number = x.Version.Number } }).ToArray()); }
public void ThreeMessages() { NestedMessage message1 = new NestedMessage.Builder { Bb = 5 }.Build(); NestedMessage message2 = new NestedMessage.Builder { Bb = 1500 }.Build(); NestedMessage message3 = new NestedMessage.Builder().Build(); byte[] data; using (MemoryStream stream = new MemoryStream()) { MessageStreamWriter<NestedMessage> writer = new MessageStreamWriter<NestedMessage>(stream); writer.Write(message1); writer.Write(message2); writer.Write(message3); writer.Flush(); data = stream.ToArray(); } TestUtil.AssertEqualBytes(ThreeMessageData, data); }
private void OnBeginAcceptTcpClient(IAsyncResult result) { TcpClient client; try { client = listener.EndAcceptTcpClient(result); listener.BeginAcceptTcpClient(OnBeginAcceptTcpClient, null); } catch (ObjectDisposedException) { return; } catch (InvalidOperationException) { return; } try { using (client) using (var stream = client.GetStream()) { var writer = new MessageStreamWriter<StorageMessageUnion>(stream); try { foreach (var wrapper in MessageStreamIterator<StorageMessageUnion>.FromStreamProvider(() => new UndisposableStream(stream))) { log.DebugFormat("Got message {0}", wrapper.Type); switch (wrapper.Type) { case StorageMessageType.GetRequests: HandleGet(wrapper, wrapper.TopologyVersion.Value, writer); break; case StorageMessageType.PutRequests: HandlePut(wrapper, wrapper.TopologyVersion.Value, writer); break; case StorageMessageType.RemoveRequests: HandleRemove(wrapper, wrapper.TopologyVersion.Value, writer); break; case StorageMessageType.AssignAllEmptySegmentsRequest: HandleAssignEmpty(wrapper, writer); break; case StorageMessageType.ReplicateNextPageRequest: HandleReplicateNextPage(wrapper, writer); break; case StorageMessageType.UpdateTopology: HandleTopologyUpdate(writer); break; default: throw new InvalidOperationException("Message type was not understood: " + wrapper.Type); } writer.Flush(); stream.Flush(); } } catch(IOException) { // disconnected, so nothing else to do } catch (SeeOtherException e) { writer.Write(new StorageMessageUnion.Builder { Type = StorageMessageType.SeeOtherError, SeeOtherError = new SeeOtherErrorMessage.Builder { Other = e.Endpoint.GetNodeEndpoint() }.Build() }.Build()); writer.Flush(); stream.Flush(); } catch (TopologyVersionDoesNotMatchException) { writer.Write(new StorageMessageUnion.Builder { Type = StorageMessageType.TopologyChangedError, }.Build()); writer.Flush(); stream.Flush(); } catch (Exception e) { log.Warn("Error performing request", e); writer.Write(new StorageMessageUnion.Builder { Type = StorageMessageType.StorageErrorResult, Exception = new ErrorMessage.Builder { Message = e.ToString() }.Build() }.Build()); writer.Flush(); stream.Flush(); } } } catch (Exception e) { log.Warn("Error when processing request to storage", e); } }
private void PersistTopology() { byte[] buffer; var topology = master.Topology.GetTopology(); using (var stream = new MemoryStream()) { var writer = new MessageStreamWriter<TopologyResultMessage>(stream); writer.Write(topology); writer.Flush(); buffer = stream.ToArray(); } hashTable.Batch(actions => { var values = actions.Get(new GetRequest { Key = Constants.Topology }); actions.Put(new PutRequest { Key = Constants.Topology, ParentVersions = values.Select(x => x.Version).ToArray(), Bytes = buffer, IsReadOnly = true }); actions.Commit(); }); }
private void OnAcceptTcpClient(IAsyncResult result) { TcpClient client; try { client = listener.EndAcceptTcpClient(result); } catch (ObjectDisposedException) { return; } //this is done intentionally in a single threaded fashion //the master is not a hot spot and it drastically simplify our life //to avoid having to do multi threaded stuff here //all calls to the master are also very short try { using (client) using (var stream = client.GetStream()) { var writer = new MessageStreamWriter<MasterMessageUnion>(stream); foreach (var wrapper in MessageStreamIterator<MasterMessageUnion>.FromStreamProvider(() => stream)) { try { log.DebugFormat("Accepting message from {0} - {1}", client.Client.RemoteEndPoint, wrapper.Type); switch (wrapper.Type) { case MasterMessageType.GetTopologyRequest: HandleGetToplogy(writer); break; case MasterMessageType.JoinRequest: HandleJoin(wrapper, writer); break; case MasterMessageType.CaughtUpRequest: HandleCatchUp(wrapper, writer); break; case MasterMessageType.GaveUpRequest: HandleGaveUp(wrapper, writer); break; default: throw new ArgumentOutOfRangeException(); } writer.Flush(); stream.Flush(); } catch (Exception e) { log.Warn("Error performing request", e); writer.Write(new MasterMessageUnion.Builder { Type = MasterMessageType.MasterErrorResult, Exception = new ErrorMessage.Builder { Message = e.ToString() }.Build() }.Build()); writer.Flush(); stream.Flush(); } } } } catch (Exception e) { log.Warn("Error when dealing with a request (or could not send error details)", e); } finally { try { listener.BeginAcceptTcpClient(OnAcceptTcpClient, null); } catch (InvalidOperationException) { //the listener was closed } } }
private void OnAcceptTcpClient(IAsyncResult result) { TcpClient client; try { client = listener.EndAcceptTcpClient(result); } catch (ObjectDisposedException) { return; } //this is done intentionally in a single threaded fashion //the master is not a hot spot and it drastically simplify our life //to avoid having to do multi threaded stuff here //all calls to the master are also very short try { using (client) using (var stream = client.GetStream()) { var writer = new MessageStreamWriter <MasterMessageUnion>(stream); foreach (var wrapper in MessageStreamIterator <MasterMessageUnion> .FromStreamProvider(() => stream)) { try { log.DebugFormat("Accepting message from {0} - {1}", client.Client.RemoteEndPoint, wrapper.Type); switch (wrapper.Type) { case MasterMessageType.GetTopologyRequest: HandleGetToplogy(writer); break; case MasterMessageType.JoinRequest: HandleJoin(wrapper, writer); break; case MasterMessageType.CaughtUpRequest: HandleCatchUp(wrapper, writer); break; case MasterMessageType.GaveUpRequest: HandleGaveUp(wrapper, writer); break; default: throw new ArgumentOutOfRangeException(); } writer.Flush(); stream.Flush(); } catch (Exception e) { log.Warn("Error performing request", e); writer.Write(new MasterMessageUnion.Builder { Type = MasterMessageType.MasterErrorResult, Exception = new ErrorMessage.Builder { Message = e.ToString() }.Build() }.Build()); writer.Flush(); stream.Flush(); } } } } catch (Exception e) { log.Warn("Error when dealing with a request (or could not send error details)", e); } finally { try { listener.BeginAcceptTcpClient(OnAcceptTcpClient, null); } catch (InvalidOperationException) { //the listener was closed } } }
private void OnBeginAcceptTcpClient(IAsyncResult result) { TcpClient client; try { client = listener.EndAcceptTcpClient(result); listener.BeginAcceptTcpClient(OnBeginAcceptTcpClient, null); } catch (ObjectDisposedException) { return; } catch (InvalidOperationException) { return; } try { using (client) using (var stream = client.GetStream()) { var writer = new MessageStreamWriter <StorageMessageUnion>(stream); try { foreach (var wrapper in MessageStreamIterator <StorageMessageUnion> .FromStreamProvider(() => new UndisposableStream(stream))) { log.DebugFormat("Got message {0}", wrapper.Type); switch (wrapper.Type) { case StorageMessageType.GetRequests: HandleGet(wrapper, wrapper.TopologyVersion.Value, writer); break; case StorageMessageType.PutRequests: HandlePut(wrapper, wrapper.TopologyVersion.Value, writer); break; case StorageMessageType.RemoveRequests: HandleRemove(wrapper, wrapper.TopologyVersion.Value, writer); break; case StorageMessageType.AssignAllEmptySegmentsRequest: HandleAssignEmpty(wrapper, writer); break; case StorageMessageType.ReplicateNextPageRequest: HandleReplicateNextPage(wrapper, writer); break; case StorageMessageType.UpdateTopology: HandleTopologyUpdate(writer); break; default: throw new InvalidOperationException("Message type was not understood: " + wrapper.Type); } writer.Flush(); stream.Flush(); } } catch (IOException) { // disconnected, so nothing else to do } catch (SeeOtherException e) { writer.Write(new StorageMessageUnion.Builder { Type = StorageMessageType.SeeOtherError, SeeOtherError = new SeeOtherErrorMessage.Builder { Other = e.Endpoint.GetNodeEndpoint() }.Build() }.Build()); writer.Flush(); stream.Flush(); } catch (TopologyVersionDoesNotMatchException) { writer.Write(new StorageMessageUnion.Builder { Type = StorageMessageType.TopologyChangedError, }.Build()); writer.Flush(); stream.Flush(); } catch (Exception e) { log.Warn("Error performing request", e); writer.Write(new StorageMessageUnion.Builder { Type = StorageMessageType.StorageErrorResult, Exception = new ErrorMessage.Builder { Message = e.ToString() }.Build() }.Build()); writer.Flush(); stream.Flush(); } } } catch (Exception e) { log.Warn("Error when processing request to storage", e); } }