private async ValueTask DoImmediateBatchIndexAck(BatchMessageId msgId, int batchIndex, int batchSize, AckType ackType, IDictionary <string, long> properties) { var cnx = await Cnx(); if (cnx == null) { Context.System.Log.Error("Consumer connect fail!"); return; } BitArray bitSet; if (msgId.Acker != null && !(msgId.Acker is BatchMessageAckerDisabled)) { bitSet = new BitArray(msgId.Acker.BatchSize, true); } else { bitSet = new BitArray(batchSize, true); } if (ackType == AckType.Cumulative) { for (var j = 0; j <= batchIndex; j++) { bitSet[j] = false; } } else { bitSet[batchIndex] = false; } await NewMessageAckCommandAndWrite(cnx, _consumerId, msgId.LedgerId, msgId.EntryId, bitSet.ToLongArray(), ackType, null, properties, true, null); }
public void DeserializationTest() { // initialize BitSet with null var ackerDisabled = new BatchMessageAcker(null, 0); var batchMsgId = new BatchMessageId(0, 0, 0, 0, 0, ackerDisabled); try { var d = JsonSerializer.Serialize(batchMsgId, new JsonSerializerOptions { IgnoreNullValues = false }); //Assert.fail("Shouldn't be deserialized"); } catch (Exception e) { // expected Assert.False(false); } // use the default BatchMessageAckerDisabled var batchMsgIdToDeserialize = new BatchMessageId(0, 0, 0, 0); try { var d = JsonSerializer.Serialize(batchMsgIdToDeserialize, new JsonSerializerOptions { IgnoreNullValues = false }); } catch (Exception ex) { Assert.False(false); } }
public virtual void TestLessThan() { var MessageId1 = new MessageId(124L, 345L, 567); var MessageId2 = new MessageId(123L, 345L, 567); var MessageId3 = new MessageId(123L, 344L, 567); var MessageId4 = new MessageId(123L, 344L, 566); var batchMessageId1 = new BatchMessageId(235L, 345L, 456, 567); var batchMessageId2 = new BatchMessageId(234L, 346L, 456, 567); var batchMessageId3 = new BatchMessageId(234L, 345L, 456, 568); var batchMessageId4 = new BatchMessageId(234L, 345L, 457, 567); var batchMessageId5 = new BatchMessageId(234L, 345L, 456, 567); Assert.True(MessageId2.CompareTo(MessageId1) < 0, "Expected to be less than"); Assert.True(MessageId3.CompareTo(MessageId1) < 0, "Expected to be less than"); Assert.True(MessageId4.CompareTo(MessageId1) < 0, "Expected to be less than"); Assert.True(MessageId3.CompareTo(MessageId2) < 0, "Expected to be less than"); Assert.True(MessageId4.CompareTo(MessageId2) < 0, "Expected to be less than"); Assert.True(MessageId4.CompareTo(MessageId3) < 0, "Expected to be less than"); Assert.True(batchMessageId2.CompareTo(batchMessageId1) < 0, "Expected to be less than"); Assert.True(batchMessageId3.CompareTo(batchMessageId1) < 0, "Expected to be less than"); Assert.True(batchMessageId4.CompareTo(batchMessageId1) < 0, "Expected to be less than"); Assert.True(batchMessageId5.CompareTo(batchMessageId1) < 0, "Expected to be less than"); Assert.True(batchMessageId3.CompareTo(batchMessageId2) < 0, "Expected to be less than"); Assert.True(batchMessageId4.CompareTo(batchMessageId2) < 0, "Expected to be less than"); Assert.True(batchMessageId5.CompareTo(batchMessageId2) < 0, "Expected to be less than"); Assert.True(batchMessageId4.CompareTo(batchMessageId3) < 0, "Expected to be less than"); Assert.True(batchMessageId5.CompareTo(batchMessageId3) < 0, "Expected to be less than"); Assert.True(batchMessageId5.CompareTo(batchMessageId4) < 0, "Expected to be less than"); }
public static IMessageId FromByteArrayWithTopic(byte[] data, TopicName topicName) { if (data == null) { throw new ArgumentException(); } var builder = new MessageIdData(); MessageIdData idData = builder; IMessageId messageId; if (idData.BatchIndex >= 0) { messageId = new BatchMessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition, idData.BatchIndex, idData.BatchSize, BatchMessageAcker.NewAcker(idData.BatchSize)); } else { messageId = new MessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition); } if (idData.Partition > -1 && topicName != null) { var t = new TopicName(); messageId = new TopicMessageId(t.GetPartition(idData.Partition).ToString(), topicName.ToString(), messageId); } return(messageId); }
public void HashCodeTest() { var batchMsgId1 = new BatchMessageId(0, 0, 0, 0); var batchMsgId2 = new BatchMessageId(1, 1, 1, 1); Assert.Equal(batchMsgId1.GetHashCode(), batchMsgId1.GetHashCode()); Assert.True(batchMsgId1.GetHashCode() != batchMsgId2.GetHashCode()); }
private MessageId ModifyBatchMessageIdAndStatesInConsumer(BatchMessageId batchMessageId) { var messageId = new MessageId(batchMessageId.LedgerId, batchMessageId.EntryId, batchMessageId.PartitionIndex); _consumer.Tell(new IncrementNumAcksSent(batchMessageId.BatchSize)); ClearMessageIdFromUnAckTrackerAndDeadLetter(messageId); return(messageId); }
public AddBatchIndexAcknowledgment(BatchMessageId messageId, int batchIndex, int batchSize, CommandAck.AckType ackType, IDictionary <string, long> properties, IActorRef txn) { MessageId = messageId; BatchIndex = batchIndex; BatchSize = batchSize; AckType = ackType; Properties = properties; Txn = txn; }
public void CompareToTest() { var batchMsgId1 = new BatchMessageId(0, 0, 0, 0); var batchMsgId2 = new BatchMessageId(1, 1, 1, 1); Assert.Equal(-1, batchMsgId1.CompareTo(batchMsgId2)); Assert.Equal(1, batchMsgId2.CompareTo(batchMsgId1)); Assert.Equal(0, batchMsgId2.CompareTo(batchMsgId2)); }
private void DoIndividualBatchAckAsync(BatchMessageId batchMessageId) { var msgId = new MessageId(batchMessageId.LedgerId, batchMessageId.EntryId, batchMessageId.PartitionIndex); if (!_pendingIndividualBatchIndexAcks.TryGetValue(msgId, out _)) { _pendingIndividualBatchIndexAcks.Add(msgId); } }
public static Message<T> Create(string topic, BatchMessageId batchMessageIdImpl, MessageMetadata batchMetadata, SingleMessageMetadata singleMessageMetadata, ReadOnlySequence<byte> payload, Option<EncryptionContext> encryptionCtx, IActorRef cnx, ISchema<T> schema, int redeliveryCount, bool pooledMessage) { if (pooledMessage) { var msg = new Message<T>(); Init(msg, topic, batchMessageIdImpl, batchMetadata, singleMessageMetadata, payload, encryptionCtx, cnx, schema, redeliveryCount, pooledMessage); return msg; } return new Message<T>(topic, batchMessageIdImpl, batchMetadata, singleMessageMetadata, payload, encryptionCtx, cnx, schema, redeliveryCount, pooledMessage); }
public virtual void TestEqual() { var MessageId1 = new MessageId(123L, 345L, 567); var MessageId2 = new MessageId(123L, 345L, 567); var batchMessageId1 = new BatchMessageId(234L, 345L, 456, 567); var batchMessageId2 = new BatchMessageId(234L, 345L, 456, 567); Assert.Equal(0, MessageId1.CompareTo(MessageId2)); Assert.Equal(0, batchMessageId1.CompareTo(batchMessageId2)); }
private async ValueTask DoIndividualBatchAck(BatchMessageId batchMessageId, IDictionary <string, long> properties) { if (_acknowledgementGroupTimeMicros == 0 || (properties != null && properties.Count > 0)) { await DoImmediateBatchIndexAck(batchMessageId, batchMessageId.BatchIndex, batchMessageId.BatchSize, AckType.Individual, properties); } else { await DoIndividualBatchAck(batchMessageId); } }
private async ValueTask DoCumulativeBatchIndexAck(BatchMessageId batchMessageId, IDictionary <string, long> properties) { if (_acknowledgementGroupTimeMicros == 0 || (properties != null && properties.Count > 0)) { await DoImmediateBatchIndexAck(batchMessageId, batchMessageId.BatchIndex, batchMessageId.BatchSize, AckType.Cumulative, properties); } else { var bitSet = new BitArray(batchMessageId.BatchSize, true); bitSet[batchMessageId.BatchSize] = false; await DoCumulativeAck(batchMessageId, null, bitSet.ToLongArray()); } }
public virtual void TestCompareDifferentType() { var MessageId = new MessageId(123L, 345L, 567); var batchMessageId1 = new BatchMessageId(123L, 345L, 566, 789); var batchMessageId2 = new BatchMessageId(123L, 345L, 567, 789); var batchMessageId3 = new BatchMessageId(MessageId); Assert.True(MessageId.CompareTo(batchMessageId1) > 0, "Expected to be greater than"); Assert.True(MessageId.CompareTo(batchMessageId2) < 0, "Expected to be less than"); Assert.Equal(0, MessageId.CompareTo(batchMessageId3)); Assert.True(batchMessageId1.CompareTo(MessageId) < 0, "Expected to be less than"); Assert.True(batchMessageId2.CompareTo(MessageId) > 0, "Expected to be greater than"); Assert.Equal(0, batchMessageId3.CompareTo(MessageId)); }
public virtual void TestBatchMessageIdCompareToTopicMessageId() { var MessageId1 = new BatchMessageId(123L, 345L, 567, 789); var MessageId2 = new BatchMessageId(123L, 345L, 567, 0); var MessageId3 = new BatchMessageId(123L, 345L, 567, -1); var topicMessageId1 = new TopicMessageId("test-topic-partition-0", "test-topic", new MessageId(123L, 345L, 566)); var topicMessageId2 = new TopicMessageId("test-topic-partition-0", "test-topic", new MessageId(123L, 345L, 567)); Assert.True(MessageId1.CompareTo(topicMessageId1) > 0, "Expected to be greater than"); Assert.True(MessageId1.CompareTo(topicMessageId2) > 0, "Expected to be greater than"); Assert.True(MessageId2.CompareTo(topicMessageId2) > 0, "Expected to be greater than"); Assert.Equal(0, MessageId3.CompareTo(topicMessageId2)); Assert.True(topicMessageId1.CompareTo(MessageId1) < 0, "Expected to be less than"); Assert.True(topicMessageId2.CompareTo(MessageId1) < 0, "Expected to be less than"); Assert.True(topicMessageId2.CompareTo(MessageId2) < 0, "Expected to be less than"); Assert.True(topicMessageId2.CompareTo(MessageId2) < 0, "Expected to be less than"); }
public virtual void CompareToSymmetricTest() { var simpleMessageId = new MessageId(123L, 345L, 567); // batchIndex is -1 if message is non-batched message and has the batchIndex for a batch message var batchMessageId1 = new BatchMessageId(123L, 345L, 567, -1); var batchMessageId2 = new BatchMessageId(123L, 345L, 567, 1); var batchMessageId3 = new BatchMessageId(123L, 345L, 566, 1); var batchMessageId4 = new BatchMessageId(123L, 345L, 566, -1); Assert.Equal(0, simpleMessageId.CompareTo(batchMessageId1)); Assert.Equal(0, batchMessageId1.CompareTo(simpleMessageId)); Assert.True(batchMessageId2.CompareTo(simpleMessageId) > 0, "Expected to be greater than"); Assert.True(simpleMessageId.CompareTo(batchMessageId2) < 0, "Expected to be less than"); Assert.True(simpleMessageId.CompareTo(batchMessageId3) > 0, "Expected to be greater than"); Assert.True(batchMessageId3.CompareTo(simpleMessageId) < 0, "Expected to be less than"); Assert.True(simpleMessageId.CompareTo(batchMessageId4) > 0, "Expected to be greater than"); Assert.True(batchMessageId4.CompareTo(simpleMessageId) < 0, "Expected to be less than"); }
private async ValueTask DoIndividualBatchAck(BatchMessageId batchMessageId) { var cnx = await Cnx(); if (await IsAckReceiptEnabled(cnx)) { try { DoIndividualBatchAckAsync(batchMessageId); } finally { } } else { DoIndividualBatchAckAsync(batchMessageId); } }
// / Serialization public static IMessageId FromByteArray(byte[] data) { if (data == null) { throw new ArgumentException(); } var inputStream = new CodedInputStream(data); var builder = new MessageIdData(); MessageIdData idData = builder; try { //idData.MergeFrom(inputStream); } catch (System.Exception e) { throw e; } MessageId messageId; if (idData.BatchIndex >= 0) { if (idData.BatchSize > 0) { messageId = new BatchMessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition, idData.BatchIndex, idData.BatchSize, BatchMessageAcker.NewAcker(idData.BatchSize)); } else { messageId = new BatchMessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition, idData.BatchIndex); } } else { messageId = new MessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition); } return(messageId); }
public void EqualsTest() { var batchMsgId1 = new BatchMessageId(0, 0, 0, 0); var batchMsgId2 = new BatchMessageId(1, 1, 1, 1); var batchMsgId3 = new BatchMessageId(0, 0, 0, 1); var batchMsgId4 = new BatchMessageId(0, 0, 0, -1); var msgId = new MessageId(0, 0, 0); Assert.True(batchMsgId1.Equals(batchMsgId1)); Assert.False(batchMsgId1.Equals(batchMsgId2)); Assert.False(batchMsgId1.Equals(batchMsgId3)); Assert.False(batchMsgId1.Equals(batchMsgId4)); Assert.False(batchMsgId1.Equals(msgId)); Assert.Equal(msgId, msgId); Assert.False(msgId.Equals(batchMsgId1)); Assert.False(msgId.Equals(batchMsgId2)); Assert.False(msgId.Equals(batchMsgId3)); Assert.Equal(msgId, batchMsgId4); Assert.Equal(batchMsgId4, msgId); }
private static void Init(Message<T> msg, string topic, BatchMessageId batchMessageIdImpl, MessageMetadata msgMetadata, SingleMessageMetadata singleMessageMetadata, ReadOnlySequence<byte> payload, Option<EncryptionContext> encryptionCtx, IActorRef cnx, ISchema<T> schema, int redeliveryCount, bool poolMessage) { //msg._metadata.Clear(); msg._metadata = new Metadata(); msg._messageId = batchMessageIdImpl; msg._topic = topic; msg._cnx = cnx; msg._redeliveryCount = redeliveryCount; msg._encryptionCtx = encryptionCtx; msg._schema = schema; msg._poolMessage = poolMessage; // If it's not pool message then need to make a copy since the passed payload is // using a ref-count buffer that we don't know when could release, since the // Message is passed to the user. Also, the passed ByteBuf is coming from network // and is backed by a direct buffer which we could not expose as a byte[] msg._payload = payload;//poolMessage ? payload.retain() : Unpooled.copiedBuffer(payload); msg._metadata.ReplicateTo = msgMetadata.ReplicateToes; if (singleMessageMetadata != null) { if (singleMessageMetadata.Properties.Count > 0) { IDictionary<string, string> properties = new Dictionary<string, string>(); foreach (var entry in singleMessageMetadata.Properties) { properties[entry.Key] = entry.Value; } msg.Properties = properties.ToImmutableDictionary(); } else { msg.Properties = new Dictionary<string, string>(); } if (singleMessageMetadata.ShouldSerializePartitionKey()) { msg._metadata.PartitionKeyB64Encoded = singleMessageMetadata.PartitionKeyB64Encoded; msg._metadata.PartitionKey = singleMessageMetadata.PartitionKey; } else if (msgMetadata.ShouldSerializePartitionKey()) { msg._metadata.PartitionKey = string.Empty; msg._metadata.PartitionKeyB64Encoded = false; } if (singleMessageMetadata.ShouldSerializeOrderingKey()) { msg._metadata.OrderingKey = singleMessageMetadata.OrderingKey; } else if (msgMetadata.ShouldSerializeOrderingKey()) { msg._metadata.OrderingKey = null; } if (singleMessageMetadata.ShouldSerializeEventTime()) { msg._metadata.EventTime = (long)singleMessageMetadata.EventTime; } if (singleMessageMetadata.ShouldSerializeSequenceId()) { msg._metadata.SequenceId = (long)singleMessageMetadata.SequenceId; } if (singleMessageMetadata.ShouldSerializeNullValue()) { msg._metadata.NullValue = singleMessageMetadata.NullValue; } if (singleMessageMetadata.ShouldSerializeNullPartitionKey()) { msg._metadata.NullPartitionKey = singleMessageMetadata.NullPartitionKey; } } else if (msgMetadata.Properties.Count > 0) { msg.Properties = msgMetadata.Properties.ToDictionary(x => x.Key, x=> x.Value).ToImmutableDictionary(); } else { msg.Properties = new Dictionary<string, string>(); } msg._metadata = EnsureMetadata(msgMetadata, msg._metadata); }
internal Message(string topic, BatchMessageId batchMessageIdImpl, MessageMetadata batchMetadata, SingleMessageMetadata singleMessageMetadata, ReadOnlySequence<byte> payload, Option<EncryptionContext> encryptionCtx, IActorRef cnx, ISchema<T> schema, int redeliveryCount, bool keepMessageInDirectMemory) { Init(this, topic, batchMessageIdImpl, batchMetadata, singleMessageMetadata, payload, encryptionCtx, cnx, schema, redeliveryCount, keepMessageInDirectMemory); }
internal Message(string topic, BatchMessageId batchMessageIdImpl, MessageMetadata msgMetadata, SingleMessageMetadata singleMessageMetadata, ReadOnlySequence<byte> payload, Option<EncryptionContext> encryptionCtx, IActorRef cnx, ISchema<T> schema) : this(topic, batchMessageIdImpl, msgMetadata, singleMessageMetadata, payload, encryptionCtx, cnx, schema, 0, false) { }