Ejemplo n.º 1
0
        public async Task should_support_out_of_order_ZebusV2_acks_and_messages()
        {
            var messageBytes = new byte[512];

            new Random().NextBytes(messageBytes);
            var timestamp = DateTime.UtcNow;
            var messageId = new MessageId(MessageIdV2.CreateNewSequentialId(timestamp.Ticks));
            var peerId    = "Abc.Peer.0";

            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Ack(new PeerId(peerId), messageId)
            });

            await Task.Delay(50);

            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Message(new PeerId(peerId), messageId, MessageTypeId.PersistenceStopping, messageBytes)
            });

            var retrievedMessage = DataContext.PersistentMessages.Execute().ExpectedSingle();

            retrievedMessage.TransportMessage.ShouldBeNull();
            retrievedMessage.BucketId.ShouldEqual(GetBucketIdFromDateTime(timestamp));
            retrievedMessage.IsAcked.ShouldBeTrue();
            retrievedMessage.PeerId.ShouldEqual(peerId);
            retrievedMessage.UniqueTimestampInTicks.ShouldEqual(timestamp.Ticks);
        }
Ejemplo n.º 2
0
        public async Task should_persist_messages_in_order()
        {
            var firstPeer  = new PeerId("Abc.Testing.Target");
            var secondPeer = new PeerId("Abc.Testing.OtherTarget");

            _peerStateRepository.Add(new PeerState(firstPeer, 0, SystemDateTime.UtcNow.Date.Ticks));
            _peerStateRepository.Add(new PeerState(secondPeer, 0, SystemDateTime.UtcNow.Date.Ticks));

            using (MessageId.PauseIdGeneration())
                using (SystemDateTime.PauseTime())
                {
                    var expectedTransportMessages = Enumerable.Range(1, 100).Select(CreateTestTransportMessage).ToList();
                    var messages = expectedTransportMessages.SelectMany(x =>
                    {
                        var transportMessageBytes = Serialization.Serializer.Serialize(x).ToArray();
                        return(new[]
                        {
                            MatcherEntry.Message(firstPeer, x.Id, x.MessageTypeId, transportMessageBytes),
                            MatcherEntry.Message(secondPeer, x.Id, x.MessageTypeId, transportMessageBytes),
                        });
                    })
                                   .ToList();

                    await _storage.Write(messages);

                    _peerStateRepository[firstPeer].NonAckedMessageCount.ShouldEqual(100);
                    _peerStateRepository[secondPeer].NonAckedMessageCount.ShouldEqual(100);

                    var readerForFirstPeer = (CqlMessageReader)_storage.CreateMessageReader(firstPeer);
                    readerForFirstPeer.GetUnackedMessages().ToList().ShouldEqualDeeply(expectedTransportMessages);

                    var readerForSecondPeer = (CqlMessageReader)_storage.CreateMessageReader(secondPeer);
                    readerForSecondPeer.GetUnackedMessages().ToList().ShouldEqualDeeply(expectedTransportMessages);
                }
        }
Ejemplo n.º 3
0
        public void should_test_replay()
        {
            var messageBytes = MessageBytes();

            // Fill with lots of unacked messages
            var entriesToPersist = new List <MatcherEntry>();
            var peerId           = new PeerId("Peer");

            for (int i = 0; i < 10_000; i++)
            {
                MessageId.PauseIdGenerationAtDate(SystemDateTime.UtcNow.Date.AddSeconds(i * 10));
                entriesToPersist.Add(MatcherEntry.Message(peerId, MessageId.NextId(), new MessageTypeId("SomeEvent"), messageBytes));
            }

            _storage.Write(entriesToPersist);

            // Read all unacked messages
            var messageReader = _storage.CreateMessageReader(peerId);
            var startTime     = DateTime.UtcNow;
            var testDuration  = 30.Seconds();
            var count         = 0;

            while (DateTime.UtcNow - startTime < testDuration)
            {
                foreach (var transportMessage in messageReader.GetUnackedMessages())
                {
                }

                count++;
            }

            Console.WriteLine($"Replayed {count:N0} times ({count*entriesToPersist.Count:N0} messages) in {testDuration.TotalSeconds:N0}s");
        }
Ejemplo n.º 4
0
        public async Task should_write_message_entry_fields_to_cassandra()
        {
            using (SystemDateTime.PauseTime())
            {
                var messageBytes = new byte[512];
                new Random().NextBytes(messageBytes);
                var messageId = MessageId.NextId();
                var peerId    = "Abc.Peer.0";

                await _storage.Write(new List <MatcherEntry> {
                    MatcherEntry.Message(new PeerId(peerId), messageId, MessageTypeId.PersistenceStopping, messageBytes)
                });

                var retrievedMessage = DataContext.PersistentMessages.Execute().ExpectedSingle();
                retrievedMessage.TransportMessage.ShouldBeEquivalentTo(messageBytes, true);
                retrievedMessage.BucketId.ShouldEqual(GetBucketIdFromMessageId(messageId));
                retrievedMessage.IsAcked.ShouldBeFalse();
                retrievedMessage.PeerId.ShouldEqual(peerId);
                retrievedMessage.UniqueTimestampInTicks.ShouldEqual(messageId.GetDateTime().Ticks);
                var writeTimeRow = DataContext.Session.Execute("SELECT WRITETIME(\"IsAcked\") FROM \"PersistentMessage\";").ExpectedSingle();
                writeTimeRow.GetValue <long>(0).ShouldEqual(ToUnixMicroSeconds(messageId.GetDateTime()));

                var peerState = DataContext.PeerStates.Execute().ExpectedSingle();
                peerState.NonAckedMessageCount.ShouldEqual(1);
                peerState.PeerId.ShouldEqual(peerId);
                peerState.OldestNonAckedMessageTimestamp.ShouldEqual(messageId.GetDateTime().Ticks - PeerState.MessagesTimeToLive.Ticks);
            }
        }
Ejemplo n.º 5
0
        private static MatcherEntry GetMatcherEntryWithValidTransportMessage(PeerId peer, int i)
        {
            var inputMessage = CreateTestTransportMessage(i);
            var messageBytes = Serialization.Serializer.Serialize(inputMessage).ToArray();
            var message1     = MatcherEntry.Message(peer, inputMessage.Id, MessageUtil.TypeId <Message1>(), messageBytes);

            return(message1);
        }
Ejemplo n.º 6
0
        public async Task should_return_cql_message_reader()
        {
            var peerId = new PeerId("PeerId");
            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Message(peerId, MessageId.NextId(), MessageTypeId.PersistenceStopping, new byte[0])
            });

            _storage.CreateMessageReader(peerId).ShouldNotBeNull();
        }
Ejemplo n.º 7
0
        public async Task should_remove_from_cassandra_when_asked_to_remove_peer()
        {
            var peerId = new PeerId("PeerId");
            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Message(peerId, MessageId.NextId(), MessageTypeId.PersistenceStopping, new byte[0])
            });

            await _storage.RemovePeer(peerId);

            DataContext.PeerStates.Execute().ShouldBeEmpty();
        }
Ejemplo n.º 8
0
        private static List <MatcherEntry> GetEntriesToPersist(byte[] messageBytes, int offset, int count = 100)
        {
            var entriesToPersist = new List <MatcherEntry>();

            for (int i = 0; i < count; i++)
            {
                MessageId.PauseIdGenerationAtDate(SystemDateTime.UtcNow.Date.AddSeconds(i * 10));
                entriesToPersist.Add(MatcherEntry.Message(new PeerId("Peer" + (i + offset)), MessageId.NextId(), new MessageTypeId("SomeEvent"), messageBytes));
            }
            return(entriesToPersist);
        }
Ejemplo n.º 9
0
        public void should_report_storage_informations()
        {
            var peer = new PeerId("peer");

            _storage.Write(new[]
            {
                MatcherEntry.Message(peer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x01, 0x02, 0x03 }),
                MatcherEntry.Message(peer, MessageId.NextId(), new MessageTypeId("Abc.Message.Fat"), new byte[] { 0x01, 0x02, 0x03, 0x04 }),
            });

            _reporterMock.Verify(r => r.AddStorageReport(2, 7, 4, "Abc.Message.Fat"));
        }
Ejemplo n.º 10
0
        public async Task should_delete_all_buckets_for_peer_when_removed()
        {
            var peerId = new PeerId("PeerId");
            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Message(peerId, MessageId.NextId(), MessageTypeId.PersistenceStopping, new byte[0])
            });

            DataContext.PersistentMessages.Execute().Count().ShouldEqual(1);

            await _storage.RemovePeer(new PeerId("PeerId"));

            DataContext.PersistentMessages.Execute().Any().ShouldBeFalse();
        }
Ejemplo n.º 11
0
        public async Task should_remove_peer()
        {
            var inputMessage = CreateTestTransportMessage(1);
            var messageBytes = Serialization.Serializer.Serialize(inputMessage).ToArray();
            var messageId    = MessageId.NextId();

            var peerId = new PeerId("Abc.Peer.0");
            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Message(peerId, messageId, MessageTypeId.PersistenceStopping, messageBytes)
            });

            await _storage.RemovePeer(peerId);

            _storage.CreateMessageReader(peerId).ShouldBeNull();
            _storage.GetNonAckedMessageCounts().ContainsKey(peerId).ShouldBeFalse();
        }
Ejemplo n.º 12
0
        public async Task should_write_message_entry_fields_to_cassandra()
        {
            var inputMessage = CreateTestTransportMessage(1);
            var messageBytes = Serialization.Serializer.Serialize(inputMessage).ToArray();
            var messageId    = MessageId.NextId();

            var peerId = new PeerId("Abc.Peer.0");
            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Message(peerId, messageId, MessageTypeId.PersistenceStopping, messageBytes)
            });

            var messages         = _storage.CreateMessageReader(peerId).GetUnackedMessages();
            var retrievedMessage = messages.Single();

            retrievedMessage.ShouldEqualDeeply(messageBytes);
        }
Ejemplo n.º 13
0
        public void should_update_non_ack_message_count()
        {
            var firstPeer  = new PeerId("Abc.Testing.Target");
            var secondPeer = new PeerId("Abc.Testing.OtherTarget");

            _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x01, 0x02, 0x03 }) });
            _storage.Write(new[] { MatcherEntry.Message(secondPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x04, 0x05, 0x06 }) });
            _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x07, 0x08, 0x09 }) });

            _peerStateRepository[firstPeer].NonAckedMessageCount.ShouldEqual(2);
            _peerStateRepository[secondPeer].NonAckedMessageCount.ShouldEqual(1);

            _storage.Write(new[] { MatcherEntry.Ack(firstPeer, MessageId.NextId()) });

            _peerStateRepository[firstPeer].NonAckedMessageCount.ShouldEqual(1);
            _peerStateRepository[secondPeer].NonAckedMessageCount.ShouldEqual(1);
        }
Ejemplo n.º 14
0
        public async Task should_not_overwrite_messages_with_same_time_component_and_different_message_id()
        {
            var messageBytes   = Serialization.Serializer.Serialize(CreateTestTransportMessage(1)).ToArray();
            var messageId      = new MessageId(Guid.Parse("0000c399-1ab0-e511-9706-ae1ea5dcf365")); // Time component @2016-01-01 00:00:00Z
            var otherMessageId = new MessageId(Guid.Parse("0000c399-1ab0-e511-9806-f1ef55aac8e9")); // Time component @2016-01-01 00:00:00Z

            var peerId = new PeerId("Abc.Peer.0");
            await _storage.Write(new List <MatcherEntry>
            {
                MatcherEntry.Message(peerId, messageId, MessageTypeId.PersistenceStopping, messageBytes),
                MatcherEntry.Message(peerId, otherMessageId, MessageTypeId.PersistenceStopping, messageBytes),
            });

            var messages = _storage.CreateMessageReader(peerId).GetUnackedMessages();

            messages.ToList().Count.ShouldEqual(2);
        }
Ejemplo n.º 15
0
        public async Task should_store_messages_in_different_buckets()
        {
            MessageId.ResetLastTimestamp();

            var firstTime = DateTime.Now;

            using (SystemDateTime.Set(firstTime))
                using (MessageId.PauseIdGenerationAtDate(firstTime))
                {
                    var peerId = new PeerId("Abc.Testing.Target");

                    var firstMessageId = MessageId.NextId();
                    await _storage.Write(new[] { MatcherEntry.Message(peerId, firstMessageId, new MessageTypeId("Abc.Message"), new byte[] { 0x01, 0x02, 0x03 }) });

                    var secondTime = firstTime.AddHours(1);
                    SystemDateTime.Set(secondTime);
                    MessageId.PauseIdGenerationAtDate(secondTime);

                    var secondMessageId = MessageId.NextId();
                    await _storage.Write(new[] { MatcherEntry.Message(peerId, secondMessageId, new MessageTypeId("Abc.OtherMessage"), new byte[] { 0x04, 0x05, 0x06 }) });

                    var persistedMessages = DataContext.PersistentMessages.Execute().OrderBy(x => x.UniqueTimestampInTicks).ToList(); // Results are only ordered withing a partition
                    persistedMessages.Count.ShouldEqual(2);

                    persistedMessages.First().ShouldHaveSamePropertiesAs(new PersistentMessage
                    {
                        BucketId               = BucketIdHelper.GetBucketId(firstTime),
                        PeerId                 = peerId.ToString(),
                        IsAcked                = false,
                        MessageId              = firstMessageId.Value,
                        TransportMessage       = new byte[] { 0x01, 0x02, 0x03 },
                        UniqueTimestampInTicks = firstTime.Ticks
                    });
                    persistedMessages.Last().ShouldHaveSamePropertiesAs(new PersistentMessage
                    {
                        BucketId               = BucketIdHelper.GetBucketId(secondTime),
                        PeerId                 = peerId.ToString(),
                        IsAcked                = false,
                        MessageId              = secondMessageId.Value,
                        TransportMessage       = new byte[] { 0x04, 0x05, 0x06 },
                        UniqueTimestampInTicks = secondTime.Ticks
                    });
                }
        }
Ejemplo n.º 16
0
        public async Task should_not_overwrite_messages_with_same_time_component_and_different_message_id()
        {
            var messageBytes = new byte[512];

            new Random().NextBytes(messageBytes);
            var messageId      = new MessageId(Guid.Parse("0000c399-1ab0-e511-9706-ae1ea5dcf365")); // Time component @2016-01-01 00:00:00Z
            var otherMessageId = new MessageId(Guid.Parse("0000c399-1ab0-e511-9806-f1ef55aac8e9")); // Time component @2016-01-01 00:00:00Z
            var peerId         = "Abc.Peer.0";

            await _storage.Write(new List <MatcherEntry>
            {
                MatcherEntry.Message(new PeerId(peerId), messageId, MessageTypeId.PersistenceStopping, messageBytes),
                MatcherEntry.Message(new PeerId(peerId), otherMessageId, MessageTypeId.PersistenceStopping, messageBytes),
            });

            var retrievedMessages = DataContext.PersistentMessages.Execute().ToList();

            retrievedMessages.Count.ShouldEqual(2);
        }
Ejemplo n.º 17
0
        public async Task should_update_non_ack_message_count()
        {
            var firstPeer  = new PeerId("Abc.Testing.Target");
            var secondPeer = new PeerId("Abc.Testing.OtherTarget");

            await _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x01, 0x02, 0x03 }) });

            await _storage.Write(new[] { MatcherEntry.Message(secondPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x04, 0x05, 0x06 }) });

            await _storage.Write(new[] { MatcherEntry.Message(firstPeer, MessageId.NextId(), new MessageTypeId("Abc.Message"), new byte[] { 0x07, 0x08, 0x09 }) });

            var nonAckedMessageCountsForUpdatedPeers = _storage.GetNonAckedMessageCounts();

            nonAckedMessageCountsForUpdatedPeers[firstPeer].ShouldEqual(2);
            nonAckedMessageCountsForUpdatedPeers[secondPeer].ShouldEqual(1);

            await _storage.Write(new[] { MatcherEntry.Ack(firstPeer, MessageId.NextId()) });

            nonAckedMessageCountsForUpdatedPeers = _storage.GetNonAckedMessageCounts();
            nonAckedMessageCountsForUpdatedPeers[firstPeer].ShouldEqual(1);
            nonAckedMessageCountsForUpdatedPeers[secondPeer].ShouldEqual(1);
        }
Ejemplo n.º 18
0
        public async Task should_load_previous_out_of_order_acks()
        {
            var peer = new PeerId("Abc.Testing.Target");

            var messageId = MessageId.NextId();
            await _storage.Write(new[] { MatcherEntry.Ack(peer, messageId) });

            _storage.Stop();

            _storage = new RocksDbStorage(_databaseDirectoryPath);
            _storage.Start();

            var message = MatcherEntry.Message(peer, messageId, MessageUtil.TypeId <Message1>(), Array.Empty <byte>());
            await _storage.Write(new[] { message });

            using (var messageReader = _storage.CreateMessageReader(peer))
            {
                messageReader.GetUnackedMessages()
                .Count()
                .ShouldEqual(0);
            }
        }
Ejemplo n.º 19
0
        public async Task should_support_out_of_order_acks_and_messages()
        {
            var inputMessage = CreateTestTransportMessage(1);
            var messageBytes = Serialization.Serializer.Serialize(inputMessage).ToArray();
            var messageId    = MessageId.NextId();

            var peerId = new PeerId("Abc.Peer.0");
            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Ack(peerId, messageId)
            });

            await Task.Delay(50);

            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Message(peerId, messageId, MessageTypeId.PersistenceStopping, messageBytes)
            });

            var messageReader = _storage.CreateMessageReader(peerId);

            messageReader.ShouldNotBeNull();
            var messages = messageReader.GetUnackedMessages().ToList();

            messages.ShouldBeEmpty();
        }
Ejemplo n.º 20
0
        public async Task should_write_ZebusV2_message_entry_fields_to_cassandra()
        {
            var messageBytes = new byte[512];

            new Random().NextBytes(messageBytes);
            var timestamp = DateTime.UtcNow;
            var messageId = new MessageId(MessageIdV2.CreateNewSequentialId(timestamp.Ticks));
            var peerId    = "Abc.Peer.0";

            await _storage.Write(new List <MatcherEntry> {
                MatcherEntry.Message(new PeerId(peerId), messageId, MessageTypeId.PersistenceStopping, messageBytes)
            });

            var retrievedMessage = DataContext.PersistentMessages.Execute().ExpectedSingle();

            retrievedMessage.TransportMessage.ShouldBeEquivalentTo(messageBytes, true);
            retrievedMessage.BucketId.ShouldEqual(GetBucketIdFromDateTime(timestamp));
            retrievedMessage.IsAcked.ShouldBeFalse();
            retrievedMessage.PeerId.ShouldEqual(peerId);
            retrievedMessage.UniqueTimestampInTicks.ShouldEqual(timestamp.Ticks);
            var writeTimeRow = DataContext.Session.Execute("SELECT WRITETIME(\"IsAcked\") FROM \"PersistentMessage\";").ExpectedSingle();

            writeTimeRow.GetValue <long>(0).ShouldEqual(ToUnixMicroSeconds(timestamp));
        }