public void TestBadCompression()
{
using (var serialized = Pool.Reserve())
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.Gzip,
Messages = new[]
{
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
// corrupt compressed data
serialized[37] = 8;
// recompute crc
var crc = (int)Crc32.Compute(serialized, 24, serialized.Length - 24);
serialized.Position = 20;
BigEndianConverter.Write(serialized, crc);
// go
serialized.Position = 4;
Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <UncompressException>());
}
}
public void TestDeserializeMessageSet(CompressionCodec codec)
{
using (var serialized = Pool.Reserve())
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = codec,
Messages = new[]
{
new Message {
Key = Key, Value = Value
},
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized.Position = 4;
var deserialized = FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers);
Assert.AreEqual(2, deserialized.Count);
foreach (var msg in deserialized)
{
Assert.AreEqual(0, msg.Offset);
CollectionAssert.AreEqual(Key, msg.Message.Key as byte[]);
CollectionAssert.AreEqual(Value, msg.Message.Value as byte[]);
}
}
}
/// <summary>
/// Handle a partition response for a given topic: filter out of range messages
/// (can happen when receiving compressed messages), update the partitions states for the
/// topic, emit all received messages on the dispatcher, and issue a new Fetch request if needed.
/// </summary>
/// <param name="topic"></param>
/// <param name="partitionResponse"></param>
/// <returns></returns>
private async Task HandleFetchPartitionResponse(string topic, FetchPartitionResponse partitionResponse)
{
var state = _partitionsStates[topic][partitionResponse.Partition];
if (!state.Active)
{
return;
}
// Filter messages under required offset, this may happen when receiving
// compressed messages. The Kafka protocol specifies it's up to the client
// to filter out those messages. We also filter messages with offset greater
// than the required stop offset if set.
foreach (
var message in
partitionResponse.Messages.Where(
message =>
message.Offset >= state.NextOffsetExpected &&
(state.StopAt < 0 || message.Offset <= state.StopAt)))
{
state.LastOffsetSeen = message.Offset;
MessageReceived(new RawKafkaRecord
{
Topic = topic,
Key = message.Message.Key,
Value = message.Message.Value,
Offset = message.Offset,
Partition = partitionResponse.Partition
});
}
ResponseMessageListPool.Release(partitionResponse.Messages);
// Stop if we have seen the last required offset
// TODO: what if we never see it?
if (state.StopAt != Offsets.Never && state.LastOffsetSeen >= state.StopAt)
{
state.Active = false;
return;
}
// Loop fetch request
if (state.LastOffsetSeen >= Offsets.Some)
{
state.NextOffsetExpected = state.LastOffsetSeen + 1;
}
await Fetch(topic, partitionResponse.Partition, state.NextOffsetExpected);
}
public void TestSerializeProduceRequest()
{
var produce = new ProduceRequest
{
Timeout = 1223,
RequiredAcks = 1,
TopicsData = new[]
{
new TopicData <PartitionData>
{
TopicName = "barbu",
PartitionsData = new[]
{
new PartitionData
{
Partition = 22,
CompressionCodec = CompressionCodec.None,
Messages = new[]
{
new Message {
Value = TheValue
}
},
}
}
},
}
};
var config = new SerializationConfig();
config.SetSerializersForTopic("barbu", new StringSerializer(), new StringSerializer());
config.SetDeserializersForTopic("barbu", new StringDeserializer(), new StringDeserializer());
using (var serialized = produce.Serialize(new ReusableMemoryStream(null), 321, ClientId, config))
{
CheckHeader(Basics.ApiKey.ProduceRequest, 0, 321, TheClientId, serialized);
Assert.AreEqual(produce.RequiredAcks, BigEndianConverter.ReadInt16(serialized));
Assert.AreEqual(produce.Timeout, BigEndianConverter.ReadInt32(serialized));
Assert.AreEqual(1, BigEndianConverter.ReadInt32(serialized)); // 1 topic data
Assert.AreEqual("barbu", Basics.DeserializeString(serialized));
Assert.AreEqual(1, BigEndianConverter.ReadInt32(serialized)); // 1 partition data
Assert.AreEqual(22, BigEndianConverter.ReadInt32(serialized));
var msgs = FetchPartitionResponse.DeserializeMessageSet(serialized, config.GetDeserializersForTopic("barbu"));
Assert.AreEqual(1, msgs.Count);
//Assert.AreEqual(TheValue, Encoding.UTF8.GetString(msgs[0].Message.Value));
Assert.AreEqual(TheValue, msgs[0].Message.Value as string);
}
}
public void TestBadCrc()
{
using (var serialized = new ReusableMemoryStream(null))
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.None,
Messages = new[]
{
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized[20] = 8; // change crc
serialized.Position = 4;
Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <CrcException>());
}
}
public void TestUnsupportedMagicByte()
{
using (var serialized = new ReusableMemoryStream(null))
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.None,
Messages = new[]
{
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized[24] = 8; // set non supported magic byte
serialized.Position = 4;
Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <UnsupportedMagicByteVersion>());
}
}
// Utility function, statically allocated closure
private static bool IsPartitionOkForClients(FetchPartitionResponse fr)
{
return(Error.IsPartitionOkForClients(fr.ErrorCode));
}
/// <summary>
/// Handle a partition response for a given topic: filter out of range messages
/// (can happen when receiving compressed messages), update the partitions states for the
/// topic, emit all received messages on the dispatcher, and issue a new Fetch request if needed.
/// </summary>
/// <param name="topic"></param>
/// <param name="partitionResponse"></param>
/// <returns></returns>
private async Task HandleFetchPartitionResponse(string topic, FetchPartitionResponse partitionResponse)
{
var state = _partitionsStates[topic][partitionResponse.Partition];
if (partitionResponse.ErrorCode == ErrorCode.OffsetOutOfRange)
{
_cluster.Logger.LogWarning(
string.Format(
"Offset {3} out of range for topic {0} / partition {1}, will read from {2} offset instead.", topic,
partitionResponse.Partition,
_configuration.OffsetOutOfRangeStrategy == Public.Offset.Earliest ? "earliest" : "latest", state.NextOffset));
state.Active = false;
StartConsume(topic, partitionResponse.Partition, (long)_configuration.OffsetOutOfRangeStrategy);
}
else
{
if (CheckActivity(state, topic, partitionResponse.Partition))
{
// Filter messages under required offset, this may happen when receiving
// compressed messages. The Kafka protocol specifies it's up to the client
// to filter out those messages. We also filter messages with offset greater
// than the required stop offset if set.
long firstRequired = state.NextOffset;
foreach (var message in
partitionResponse.Messages.Where(
message =>
message.Offset >= firstRequired && (state.StopAt < 0 || message.Offset <= state.StopAt))
)
{
MessageReceived(new RawKafkaRecord
{
Topic = topic,
Key = message.Message.Key,
Value = message.Message.Value,
Offset = message.Offset,
Lag = partitionResponse.HighWatermarkOffset - message.Offset,
Partition = partitionResponse.Partition,
Timestamp = Timestamp.FromUnixTimestamp(message.Message.TimeStamp)
});
state.NextOffset = message.Offset + 1;
await CheckHeartbeat();
await CheckCommit(null);
// Recheck status (may have changed if heartbeat triggered rebalance)
// TODO: is it really useful to check for heartbeat/commit after each message ?
// It's merely a defense versus bad message handler, maybe we don't care and
// just better check only at the beginning of partition processing.
if (!CheckActivity(state, topic, partitionResponse.Partition))
{
break;
}
}
}
ResponseMessageListPool.Release(partitionResponse.Messages);
await CheckCommit(null);
// Stop if we have seen the last required offset
// TODO: what if we never see it?
if (!state.Active || (state.StopAt != Offsets.Never && state.NextOffset > state.StopAt))
{
state.Active = false;
return;
}
// Loop fetch request
await Fetch(topic, partitionResponse.Partition, state.NextOffset);
}
}
