private async ValueTask DoImmediateBatchIndexAck(BatchMessageId msgId, int batchIndex, int batchSize, AckType ackType, IDictionary <string, long> properties)
{
var cnx = await Cnx();
if (cnx == null)
{
Context.System.Log.Error("Consumer connect fail!");
return;
}
BitArray bitSet;
if (msgId.Acker != null && !(msgId.Acker is BatchMessageAckerDisabled))
{
bitSet = new BitArray(msgId.Acker.BatchSize, true);
}
else
{
bitSet = new BitArray(batchSize, true);
}
if (ackType == AckType.Cumulative)
{
for (var j = 0; j <= batchIndex; j++)
{
bitSet[j] = false;
}
}
else
{
bitSet[batchIndex] = false;
}
await NewMessageAckCommandAndWrite(cnx, _consumerId, msgId.LedgerId, msgId.EntryId, bitSet.ToLongArray(), ackType, null, properties, true, null);
}
public void DeserializationTest()
{
// initialize BitSet with null
var ackerDisabled = new BatchMessageAcker(null, 0);
var batchMsgId = new BatchMessageId(0, 0, 0, 0, 0, ackerDisabled);
try
{
var d = JsonSerializer.Serialize(batchMsgId, new JsonSerializerOptions {
IgnoreNullValues = false
});
//Assert.fail("Shouldn't be deserialized");
}
catch (Exception e)
{
// expected
Assert.False(false);
}
// use the default BatchMessageAckerDisabled
var batchMsgIdToDeserialize = new BatchMessageId(0, 0, 0, 0);
try
{
var d = JsonSerializer.Serialize(batchMsgIdToDeserialize, new JsonSerializerOptions {
IgnoreNullValues = false
});
}
catch (Exception ex)
{
Assert.False(false);
}
}
public virtual void TestLessThan()
{
var MessageId1 = new MessageId(124L, 345L, 567);
var MessageId2 = new MessageId(123L, 345L, 567);
var MessageId3 = new MessageId(123L, 344L, 567);
var MessageId4 = new MessageId(123L, 344L, 566);
var batchMessageId1 = new BatchMessageId(235L, 345L, 456, 567);
var batchMessageId2 = new BatchMessageId(234L, 346L, 456, 567);
var batchMessageId3 = new BatchMessageId(234L, 345L, 456, 568);
var batchMessageId4 = new BatchMessageId(234L, 345L, 457, 567);
var batchMessageId5 = new BatchMessageId(234L, 345L, 456, 567);
Assert.True(MessageId2.CompareTo(MessageId1) < 0, "Expected to be less than");
Assert.True(MessageId3.CompareTo(MessageId1) < 0, "Expected to be less than");
Assert.True(MessageId4.CompareTo(MessageId1) < 0, "Expected to be less than");
Assert.True(MessageId3.CompareTo(MessageId2) < 0, "Expected to be less than");
Assert.True(MessageId4.CompareTo(MessageId2) < 0, "Expected to be less than");
Assert.True(MessageId4.CompareTo(MessageId3) < 0, "Expected to be less than");
Assert.True(batchMessageId2.CompareTo(batchMessageId1) < 0, "Expected to be less than");
Assert.True(batchMessageId3.CompareTo(batchMessageId1) < 0, "Expected to be less than");
Assert.True(batchMessageId4.CompareTo(batchMessageId1) < 0, "Expected to be less than");
Assert.True(batchMessageId5.CompareTo(batchMessageId1) < 0, "Expected to be less than");
Assert.True(batchMessageId3.CompareTo(batchMessageId2) < 0, "Expected to be less than");
Assert.True(batchMessageId4.CompareTo(batchMessageId2) < 0, "Expected to be less than");
Assert.True(batchMessageId5.CompareTo(batchMessageId2) < 0, "Expected to be less than");
Assert.True(batchMessageId4.CompareTo(batchMessageId3) < 0, "Expected to be less than");
Assert.True(batchMessageId5.CompareTo(batchMessageId3) < 0, "Expected to be less than");
Assert.True(batchMessageId5.CompareTo(batchMessageId4) < 0, "Expected to be less than");
}
public static IMessageId FromByteArrayWithTopic(byte[] data, TopicName topicName)
{
if (data == null)
{
throw new ArgumentException();
}
var builder = new MessageIdData();
MessageIdData idData = builder;
IMessageId messageId;
if (idData.BatchIndex >= 0)
{
messageId = new BatchMessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition, idData.BatchIndex, idData.BatchSize, BatchMessageAcker.NewAcker(idData.BatchSize));
}
else
{
messageId = new MessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition);
}
if (idData.Partition > -1 && topicName != null)
{
var t = new TopicName();
messageId = new TopicMessageId(t.GetPartition(idData.Partition).ToString(), topicName.ToString(), messageId);
}
return(messageId);
}
public void HashCodeTest()
{
var batchMsgId1 = new BatchMessageId(0, 0, 0, 0);
var batchMsgId2 = new BatchMessageId(1, 1, 1, 1);
Assert.Equal(batchMsgId1.GetHashCode(), batchMsgId1.GetHashCode());
Assert.True(batchMsgId1.GetHashCode() != batchMsgId2.GetHashCode());
}
private MessageId ModifyBatchMessageIdAndStatesInConsumer(BatchMessageId batchMessageId)
{
var messageId = new MessageId(batchMessageId.LedgerId, batchMessageId.EntryId, batchMessageId.PartitionIndex);
_consumer.Tell(new IncrementNumAcksSent(batchMessageId.BatchSize));
ClearMessageIdFromUnAckTrackerAndDeadLetter(messageId);
return(messageId);
}
public AddBatchIndexAcknowledgment(BatchMessageId messageId, int batchIndex, int batchSize, CommandAck.AckType ackType, IDictionary <string, long> properties, IActorRef txn)
{
MessageId = messageId;
BatchIndex = batchIndex;
BatchSize = batchSize;
AckType = ackType;
Properties = properties;
Txn = txn;
}
public void CompareToTest()
{
var batchMsgId1 = new BatchMessageId(0, 0, 0, 0);
var batchMsgId2 = new BatchMessageId(1, 1, 1, 1);
Assert.Equal(-1, batchMsgId1.CompareTo(batchMsgId2));
Assert.Equal(1, batchMsgId2.CompareTo(batchMsgId1));
Assert.Equal(0, batchMsgId2.CompareTo(batchMsgId2));
}
private void DoIndividualBatchAckAsync(BatchMessageId batchMessageId)
{
var msgId = new MessageId(batchMessageId.LedgerId, batchMessageId.EntryId, batchMessageId.PartitionIndex);
if (!_pendingIndividualBatchIndexAcks.TryGetValue(msgId, out _))
{
_pendingIndividualBatchIndexAcks.Add(msgId);
}
}
public static Message<T> Create(string topic, BatchMessageId batchMessageIdImpl, MessageMetadata batchMetadata, SingleMessageMetadata singleMessageMetadata, ReadOnlySequence<byte> payload, Option<EncryptionContext> encryptionCtx, IActorRef cnx, ISchema<T> schema, int redeliveryCount, bool pooledMessage)
{
if (pooledMessage)
{
var msg = new Message<T>();
Init(msg, topic, batchMessageIdImpl, batchMetadata, singleMessageMetadata, payload, encryptionCtx, cnx, schema, redeliveryCount, pooledMessage);
return msg;
}
return new Message<T>(topic, batchMessageIdImpl, batchMetadata, singleMessageMetadata, payload, encryptionCtx, cnx, schema, redeliveryCount, pooledMessage);
}
public virtual void TestEqual()
{
var MessageId1 = new MessageId(123L, 345L, 567);
var MessageId2 = new MessageId(123L, 345L, 567);
var batchMessageId1 = new BatchMessageId(234L, 345L, 456, 567);
var batchMessageId2 = new BatchMessageId(234L, 345L, 456, 567);
Assert.Equal(0, MessageId1.CompareTo(MessageId2));
Assert.Equal(0, batchMessageId1.CompareTo(batchMessageId2));
}
private async ValueTask DoIndividualBatchAck(BatchMessageId batchMessageId, IDictionary <string, long> properties)
{
if (_acknowledgementGroupTimeMicros == 0 || (properties != null && properties.Count > 0))
{
await DoImmediateBatchIndexAck(batchMessageId, batchMessageId.BatchIndex, batchMessageId.BatchSize, AckType.Individual, properties);
}
else
{
await DoIndividualBatchAck(batchMessageId);
}
}
private async ValueTask DoCumulativeBatchIndexAck(BatchMessageId batchMessageId, IDictionary <string, long> properties)
{
if (_acknowledgementGroupTimeMicros == 0 || (properties != null && properties.Count > 0))
{
await DoImmediateBatchIndexAck(batchMessageId, batchMessageId.BatchIndex, batchMessageId.BatchSize, AckType.Cumulative, properties);
}
else
{
var bitSet = new BitArray(batchMessageId.BatchSize, true);
bitSet[batchMessageId.BatchSize] = false;
await DoCumulativeAck(batchMessageId, null, bitSet.ToLongArray());
}
}
public virtual void TestCompareDifferentType()
{
var MessageId = new MessageId(123L, 345L, 567);
var batchMessageId1 = new BatchMessageId(123L, 345L, 566, 789);
var batchMessageId2 = new BatchMessageId(123L, 345L, 567, 789);
var batchMessageId3 = new BatchMessageId(MessageId);
Assert.True(MessageId.CompareTo(batchMessageId1) > 0, "Expected to be greater than");
Assert.True(MessageId.CompareTo(batchMessageId2) < 0, "Expected to be less than");
Assert.Equal(0, MessageId.CompareTo(batchMessageId3));
Assert.True(batchMessageId1.CompareTo(MessageId) < 0, "Expected to be less than");
Assert.True(batchMessageId2.CompareTo(MessageId) > 0, "Expected to be greater than");
Assert.Equal(0, batchMessageId3.CompareTo(MessageId));
}
public virtual void TestBatchMessageIdCompareToTopicMessageId()
{
var MessageId1 = new BatchMessageId(123L, 345L, 567, 789);
var MessageId2 = new BatchMessageId(123L, 345L, 567, 0);
var MessageId3 = new BatchMessageId(123L, 345L, 567, -1);
var topicMessageId1 = new TopicMessageId("test-topic-partition-0", "test-topic", new MessageId(123L, 345L, 566));
var topicMessageId2 = new TopicMessageId("test-topic-partition-0", "test-topic", new MessageId(123L, 345L, 567));
Assert.True(MessageId1.CompareTo(topicMessageId1) > 0, "Expected to be greater than");
Assert.True(MessageId1.CompareTo(topicMessageId2) > 0, "Expected to be greater than");
Assert.True(MessageId2.CompareTo(topicMessageId2) > 0, "Expected to be greater than");
Assert.Equal(0, MessageId3.CompareTo(topicMessageId2));
Assert.True(topicMessageId1.CompareTo(MessageId1) < 0, "Expected to be less than");
Assert.True(topicMessageId2.CompareTo(MessageId1) < 0, "Expected to be less than");
Assert.True(topicMessageId2.CompareTo(MessageId2) < 0, "Expected to be less than");
Assert.True(topicMessageId2.CompareTo(MessageId2) < 0, "Expected to be less than");
}
public virtual void CompareToSymmetricTest()
{
var simpleMessageId = new MessageId(123L, 345L, 567);
// batchIndex is -1 if message is non-batched message and has the batchIndex for a batch message
var batchMessageId1 = new BatchMessageId(123L, 345L, 567, -1);
var batchMessageId2 = new BatchMessageId(123L, 345L, 567, 1);
var batchMessageId3 = new BatchMessageId(123L, 345L, 566, 1);
var batchMessageId4 = new BatchMessageId(123L, 345L, 566, -1);
Assert.Equal(0, simpleMessageId.CompareTo(batchMessageId1));
Assert.Equal(0, batchMessageId1.CompareTo(simpleMessageId));
Assert.True(batchMessageId2.CompareTo(simpleMessageId) > 0, "Expected to be greater than");
Assert.True(simpleMessageId.CompareTo(batchMessageId2) < 0, "Expected to be less than");
Assert.True(simpleMessageId.CompareTo(batchMessageId3) > 0, "Expected to be greater than");
Assert.True(batchMessageId3.CompareTo(simpleMessageId) < 0, "Expected to be less than");
Assert.True(simpleMessageId.CompareTo(batchMessageId4) > 0, "Expected to be greater than");
Assert.True(batchMessageId4.CompareTo(simpleMessageId) < 0, "Expected to be less than");
}
private async ValueTask DoIndividualBatchAck(BatchMessageId batchMessageId)
{
var cnx = await Cnx();
if (await IsAckReceiptEnabled(cnx))
{
try
{
DoIndividualBatchAckAsync(batchMessageId);
}
finally
{
}
}
else
{
DoIndividualBatchAckAsync(batchMessageId);
}
}
// / Serialization
public static IMessageId FromByteArray(byte[] data)
{
if (data == null)
{
throw new ArgumentException();
}
var inputStream = new CodedInputStream(data);
var builder = new MessageIdData();
MessageIdData idData = builder;
try
{
//idData.MergeFrom(inputStream);
}
catch (System.Exception e)
{
throw e;
}
MessageId messageId;
if (idData.BatchIndex >= 0)
{
if (idData.BatchSize > 0)
{
messageId = new BatchMessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition, idData.BatchIndex, idData.BatchSize, BatchMessageAcker.NewAcker(idData.BatchSize));
}
else
{
messageId = new BatchMessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition, idData.BatchIndex);
}
}
else
{
messageId = new MessageId((long)idData.ledgerId, (long)idData.entryId, idData.Partition);
}
return(messageId);
}
public void EqualsTest()
{
var batchMsgId1 = new BatchMessageId(0, 0, 0, 0);
var batchMsgId2 = new BatchMessageId(1, 1, 1, 1);
var batchMsgId3 = new BatchMessageId(0, 0, 0, 1);
var batchMsgId4 = new BatchMessageId(0, 0, 0, -1);
var msgId = new MessageId(0, 0, 0);
Assert.True(batchMsgId1.Equals(batchMsgId1));
Assert.False(batchMsgId1.Equals(batchMsgId2));
Assert.False(batchMsgId1.Equals(batchMsgId3));
Assert.False(batchMsgId1.Equals(batchMsgId4));
Assert.False(batchMsgId1.Equals(msgId));
Assert.Equal(msgId, msgId);
Assert.False(msgId.Equals(batchMsgId1));
Assert.False(msgId.Equals(batchMsgId2));
Assert.False(msgId.Equals(batchMsgId3));
Assert.Equal(msgId, batchMsgId4);
Assert.Equal(batchMsgId4, msgId);
}
private static void Init(Message<T> msg, string topic, BatchMessageId batchMessageIdImpl, MessageMetadata msgMetadata, SingleMessageMetadata singleMessageMetadata, ReadOnlySequence<byte> payload, Option<EncryptionContext> encryptionCtx, IActorRef cnx, ISchema<T> schema, int redeliveryCount, bool poolMessage)
{
//msg._metadata.Clear();
msg._metadata = new Metadata();
msg._messageId = batchMessageIdImpl;
msg._topic = topic;
msg._cnx = cnx;
msg._redeliveryCount = redeliveryCount;
msg._encryptionCtx = encryptionCtx;
msg._schema = schema;
msg._poolMessage = poolMessage;
// If it's not pool message then need to make a copy since the passed payload is
// using a ref-count buffer that we don't know when could release, since the
// Message is passed to the user. Also, the passed ByteBuf is coming from network
// and is backed by a direct buffer which we could not expose as a byte[]
msg._payload = payload;//poolMessage ? payload.retain() : Unpooled.copiedBuffer(payload);
msg._metadata.ReplicateTo = msgMetadata.ReplicateToes;
if (singleMessageMetadata != null)
{
if (singleMessageMetadata.Properties.Count > 0)
{
IDictionary<string, string> properties = new Dictionary<string, string>();
foreach (var entry in singleMessageMetadata.Properties)
{
properties[entry.Key] = entry.Value;
}
msg.Properties = properties.ToImmutableDictionary();
}
else
{
msg.Properties = new Dictionary<string, string>();
}
if (singleMessageMetadata.ShouldSerializePartitionKey())
{
msg._metadata.PartitionKeyB64Encoded = singleMessageMetadata.PartitionKeyB64Encoded;
msg._metadata.PartitionKey = singleMessageMetadata.PartitionKey;
}
else if (msgMetadata.ShouldSerializePartitionKey())
{
msg._metadata.PartitionKey = string.Empty;
msg._metadata.PartitionKeyB64Encoded = false;
}
if (singleMessageMetadata.ShouldSerializeOrderingKey())
{
msg._metadata.OrderingKey = singleMessageMetadata.OrderingKey;
}
else if (msgMetadata.ShouldSerializeOrderingKey())
{
msg._metadata.OrderingKey = null;
}
if (singleMessageMetadata.ShouldSerializeEventTime())
{
msg._metadata.EventTime = (long)singleMessageMetadata.EventTime;
}
if (singleMessageMetadata.ShouldSerializeSequenceId())
{
msg._metadata.SequenceId = (long)singleMessageMetadata.SequenceId;
}
if (singleMessageMetadata.ShouldSerializeNullValue())
{
msg._metadata.NullValue = singleMessageMetadata.NullValue;
}
if (singleMessageMetadata.ShouldSerializeNullPartitionKey())
{
msg._metadata.NullPartitionKey = singleMessageMetadata.NullPartitionKey;
}
}
else if (msgMetadata.Properties.Count > 0)
{
msg.Properties = msgMetadata.Properties.ToDictionary(x => x.Key, x=> x.Value).ToImmutableDictionary();
}
else
{
msg.Properties = new Dictionary<string, string>();
}
msg._metadata = EnsureMetadata(msgMetadata, msg._metadata);
}
internal Message(string topic, BatchMessageId batchMessageIdImpl, MessageMetadata batchMetadata, SingleMessageMetadata singleMessageMetadata, ReadOnlySequence<byte> payload, Option<EncryptionContext> encryptionCtx, IActorRef cnx, ISchema<T> schema, int redeliveryCount, bool keepMessageInDirectMemory)
{
Init(this, topic, batchMessageIdImpl, batchMetadata, singleMessageMetadata, payload, encryptionCtx, cnx, schema, redeliveryCount, keepMessageInDirectMemory);
}
internal Message(string topic, BatchMessageId batchMessageIdImpl, MessageMetadata msgMetadata, SingleMessageMetadata singleMessageMetadata, ReadOnlySequence<byte> payload, Option<EncryptionContext> encryptionCtx, IActorRef cnx, ISchema<T> schema) : this(topic, batchMessageIdImpl, msgMetadata, singleMessageMetadata, payload, encryptionCtx, cnx, schema, 0, false)
{
}
