public async Task <OperateResult> SendAsync(TransportMessage message) { var producer = _connectionPool.RentProducer(); try { var headers = new Confluent.Kafka.Headers(); foreach (var header in message.Headers) { headers.Add(header.Value != null ? new Header(header.Key, Encoding.UTF8.GetBytes(header.Value)) : new Header(header.Key, null)); } var result = await producer.ProduceAsync(message.GetName(), new Message <string, byte[]> { Headers = headers, Key = message.Headers.TryGetValue(KafkaHeaders.KafkaKey, out string kafkaMessageKey) && !string.IsNullOrEmpty(kafkaMessageKey) ? kafkaMessageKey : message.GetId(), Value = message.Body }); if (result.Status == PersistenceStatus.Persisted || result.Status == PersistenceStatus.PossiblyPersisted) { _logger.LogDebug($"kafka topic message [{message.GetName()}] has been published."); return(OperateResult.Success); } throw new PublisherSentFailedException("kafka message persisted failed!"); }
public async Task <OperateResult> SendAsync(TransportMessage message) { var producer = _connectionPool.RentProducer(); try { var headers = new Confluent.Kafka.Headers(); foreach (var header in message.Headers) { headers.Add(header.Value != null ? new Header(header.Key, Encoding.UTF8.GetBytes(header.Value)) : new Header(header.Key, null)); } var result = await producer.ProduceAsync(message.GetName(), new Message <string, byte[]> { Headers = headers, Key = message.GetId(), Value = message.Body }); if (result.Status == PersistenceStatus.Persisted || result.Status == PersistenceStatus.PossiblyPersisted) { _logger.LogDebug($"kafka topic message [{message.GetName()}] has been published."); return(OperateResult.Success); } var ex = new PublisherSentFailedException("kafka message persisted failed!"); return(OperateResult.Failed(ex)); } catch (Exception ex) { var wapperEx = new PublisherSentFailedException(ex.Message, ex); return(OperateResult.Failed(wapperEx)); } finally { var returned = _connectionPool.Return(producer); if (!returned) { producer.Dispose(); } } }
private static Message <byte[], byte[]> CreateMessage(IMessageContext context) { var headers = new Confluent.Kafka.Headers(); foreach (var header in context.Headers) { headers.Add(header.Value != null ? new Header(header.Key, header.Value) : new Header(header.Key, null)); } return(new Message <byte[], byte[]> { Key = context.PartitionKey, Value = GetMessageContent(context), Headers = headers, Timestamp = Timestamp.Default }); }
public void GetKafkaHeader_ShouldReturnKafkaHeaders() { // Arrange var kafkaHeaders = new Headers { { key, this.value } }; var messageHeaders = new MessageHeaders(); messageHeaders.Add(key, this.value); var result = new Confluent.Kafka.Headers(); foreach (var header in messageHeaders) { result.Add(header.Value != null ? new Header(header.Key, header.Value) : new Header(header.Key, null)); } // Assert result.Should().BeEquivalentTo(kafkaHeaders); }
private ConsumeResult <K, V> ConsumeImpl <K, V>( int millisecondsTimeout, IDeserializer <K> keyDeserializer, IDeserializer <V> valueDeserializer) { var msgPtr = kafkaHandle.ConsumerPoll((IntPtr)millisecondsTimeout); if (msgPtr == IntPtr.Zero) { return(null); } try { var msg = Util.Marshal.PtrToStructure <rd_kafka_message>(msgPtr); string topic = null; if (this.enableTopicNameMarshaling) { if (msg.rkt != IntPtr.Zero) { topic = Util.Marshal.PtrToStringUTF8(Librdkafka.topic_name(msg.rkt)); } } if (msg.err == ErrorCode.Local_PartitionEOF) { return(new ConsumeResult <K, V> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = null, IsPartitionEOF = true }); } long timestampUnix = 0; IntPtr timestampType = (IntPtr)TimestampType.NotAvailable; if (enableTimestampMarshaling) { timestampUnix = Librdkafka.message_timestamp(msgPtr, out timestampType); } var timestamp = new Timestamp(timestampUnix, (TimestampType)timestampType); Headers headers = null; if (enableHeaderMarshaling) { headers = new Headers(); Librdkafka.message_headers(msgPtr, out IntPtr hdrsPtr); if (hdrsPtr != IntPtr.Zero) { for (var i = 0; ; ++i) { var err = Librdkafka.header_get_all(hdrsPtr, (IntPtr)i, out IntPtr namep, out IntPtr valuep, out IntPtr sizep); if (err != ErrorCode.NoError) { break; } var headerName = Util.Marshal.PtrToStringUTF8(namep); byte[] headerValue = null; if (valuep != IntPtr.Zero) { headerValue = new byte[(int)sizep]; Marshal.Copy(valuep, headerValue, 0, (int)sizep); } headers.Add(headerName, headerValue); } } } if (msg.err != ErrorCode.NoError) { throw new ConsumeException( new ConsumeResult <byte[], byte[]> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = new Message <byte[], byte[]> { Timestamp = timestamp, Headers = headers, Key = KeyAsByteArray(msg), Value = ValueAsByteArray(msg) }, IsPartitionEOF = false }, kafkaHandle.CreatePossiblyFatalError(msg.err, null)); } K key; try { unsafe { key = keyDeserializer.Deserialize( msg.key == IntPtr.Zero ? ReadOnlySpan <byte> .Empty : new ReadOnlySpan <byte>(msg.key.ToPointer(), (int)msg.key_len), msg.key == IntPtr.Zero, new SerializationContext(MessageComponentType.Key, topic)); } } catch (Exception ex) { throw new ConsumeException( new ConsumeResult <byte[], byte[]> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = new Message <byte[], byte[]> { Timestamp = timestamp, Headers = headers, Key = KeyAsByteArray(msg), Value = ValueAsByteArray(msg) }, IsPartitionEOF = false }, new Error(ErrorCode.Local_KeyDeserialization), ex); } V val; try { unsafe { val = valueDeserializer.Deserialize( msg.val == IntPtr.Zero ? ReadOnlySpan <byte> .Empty : new ReadOnlySpan <byte>(msg.val.ToPointer(), (int)msg.len), msg.val == IntPtr.Zero, new SerializationContext(MessageComponentType.Value, topic)); } } catch (Exception ex) { throw new ConsumeException( new ConsumeResult <byte[], byte[]> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = new Message <byte[], byte[]> { Timestamp = timestamp, Headers = headers, Key = KeyAsByteArray(msg), Value = ValueAsByteArray(msg) }, IsPartitionEOF = false }, new Error(ErrorCode.Local_ValueDeserialization), ex); } return(new ConsumeResult <K, V> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = new Message <K, V> { Timestamp = timestamp, Headers = headers, Key = key, Value = val }, IsPartitionEOF = false }); } finally { Librdkafka.message_destroy(msgPtr); } }
private void DeliveryReportCallbackImpl(IntPtr rk, IntPtr rkmessage, IntPtr opaque) { // Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios). if (ownedKafkaHandle.IsClosed) { return; } var msg = Util.Marshal.PtrToStructure <rd_kafka_message>(rkmessage); // the msg._private property has dual purpose. Here, it is an opaque pointer set // by Topic.Produce to be an IDeliveryHandler. When Consuming, it's for internal // use (hence the name). if (msg._private == IntPtr.Zero) { // Note: this can occur if the ProduceAsync overload that accepts a DeliveryHandler // was used and the delivery handler was set to null. return; } var gch = GCHandle.FromIntPtr(msg._private); var deliveryHandler = (IDeliveryHandler)gch.Target; gch.Free(); Headers headers = null; if (this.enableDeliveryReportHeaders) { headers = new Headers(); Librdkafka.message_headers(rkmessage, out IntPtr hdrsPtr); if (hdrsPtr != IntPtr.Zero) { for (var i = 0; ; ++i) { var err = Librdkafka.header_get_all(hdrsPtr, (IntPtr)i, out IntPtr namep, out IntPtr valuep, out IntPtr sizep); if (err != ErrorCode.NoError) { break; } var headerName = Util.Marshal.PtrToStringUTF8(namep); byte[] headerValue = null; if (valuep != IntPtr.Zero) { headerValue = new byte[(int)sizep]; Marshal.Copy(valuep, headerValue, 0, (int)sizep); } headers.Add(headerName, headerValue); } } } IntPtr timestampType = (IntPtr)TimestampType.NotAvailable; long timestamp = 0; if (enableDeliveryReportTimestamp) { timestamp = Librdkafka.message_timestamp(rkmessage, out timestampType); } PersistenceStatus messageStatus = PersistenceStatus.PossiblyPersisted; if (enableDeliveryReportPersistedStatus) { messageStatus = Librdkafka.message_status(rkmessage); } deliveryHandler.HandleDeliveryReport( new DeliveryReport <Null, Null> { // Topic is not set here in order to avoid the marshalling cost. // Instead, the delivery handler is expected to cache the topic string. Partition = msg.partition, Offset = msg.offset, Error = KafkaHandle.CreatePossiblyFatalError(msg.err, null), Status = messageStatus, Message = new Message <Null, Null> { Timestamp = new Timestamp(timestamp, (TimestampType)timestampType), Headers = headers } } ); }
/// <inheritdoc/> public void Produce( TopicPartition topicPartition, Message <TKey, TValue> message, Action <DeliveryReport <TKey, TValue> > deliveryHandler = null) { if (deliveryHandler != null && !enableDeliveryReports) { throw new InvalidOperationException("A delivery handler was specified, but delivery reports are disabled."); } Headers headers = message.Headers ?? new Headers(); byte[] keyBytes; try { keyBytes = (keySerializer != null) ? keySerializer.Serialize(message.Key, new SerializationContext(MessageComponentType.Key, topicPartition.Topic, headers)) : throw new InvalidOperationException("Produce called with an IAsyncSerializer key serializer configured but an ISerializer is required."); } catch (Exception ex) { throw new ProduceException <TKey, TValue>( new Error(ErrorCode.Local_KeySerialization, ex.ToString()), new DeliveryResult <TKey, TValue> { Message = message, TopicPartitionOffset = new TopicPartitionOffset(topicPartition, Offset.Unset), } ); } byte[] valBytes; try { valBytes = (valueSerializer != null) ? valueSerializer.Serialize(message.Value, new SerializationContext(MessageComponentType.Value, topicPartition.Topic, headers)) : throw new InvalidOperationException("Produce called with an IAsyncSerializer value serializer configured but an ISerializer is required."); } catch (Exception ex) { throw new ProduceException <TKey, TValue>( new Error(ErrorCode.Local_ValueSerialization, ex.ToString()), new DeliveryResult <TKey, TValue> { Message = message, TopicPartitionOffset = new TopicPartitionOffset(topicPartition, Offset.Unset), } ); } try { ProduceImpl( topicPartition.Topic, valBytes, 0, valBytes == null ? 0 : valBytes.Length, keyBytes, 0, keyBytes == null ? 0 : keyBytes.Length, message.Timestamp, topicPartition.Partition, headers, new TypedDeliveryHandlerShim_Action( topicPartition.Topic, enableDeliveryReportKey ? message.Key : default(TKey), enableDeliveryReportValue ? message.Value : default(TValue), deliveryHandler)); } catch (KafkaException ex) { throw new ProduceException <TKey, TValue>( ex.Error, new DeliveryReport <TKey, TValue> { Message = message, TopicPartitionOffset = new TopicPartitionOffset(topicPartition, Offset.Unset) }); } }
/// <inheritdoc/> public async Task <DeliveryResult <TKey, TValue> > ProduceAsync( TopicPartition topicPartition, Message <TKey, TValue> message, CancellationToken cancellationToken) { Headers headers = message.Headers ?? new Headers(); byte[] keyBytes; try { keyBytes = (keySerializer != null) ? keySerializer.Serialize(message.Key, new SerializationContext(MessageComponentType.Key, topicPartition.Topic, headers)) : await asyncKeySerializer.SerializeAsync(message.Key, new SerializationContext(MessageComponentType.Key, topicPartition.Topic, headers)).ConfigureAwait(false); } catch (Exception ex) { throw new ProduceException <TKey, TValue>( new Error(ErrorCode.Local_KeySerialization), new DeliveryResult <TKey, TValue> { Message = message, TopicPartitionOffset = new TopicPartitionOffset(topicPartition, Offset.Unset) }, ex); } byte[] valBytes; try { valBytes = (valueSerializer != null) ? valueSerializer.Serialize(message.Value, new SerializationContext(MessageComponentType.Value, topicPartition.Topic, headers)) : await asyncValueSerializer.SerializeAsync(message.Value, new SerializationContext(MessageComponentType.Value, topicPartition.Topic, headers)).ConfigureAwait(false); } catch (Exception ex) { throw new ProduceException <TKey, TValue>( new Error(ErrorCode.Local_ValueSerialization), new DeliveryResult <TKey, TValue> { Message = message, TopicPartitionOffset = new TopicPartitionOffset(topicPartition, Offset.Unset) }, ex); } try { if (enableDeliveryReports) { var handler = new TypedTaskDeliveryHandlerShim( topicPartition.Topic, enableDeliveryReportKey ? message.Key : default(TKey), enableDeliveryReportValue ? message.Value : default(TValue)); if (cancellationToken != null && cancellationToken.CanBeCanceled) { handler.CancellationTokenRegistration = cancellationToken.Register(() => handler.TrySetCanceled()); } ProduceImpl( topicPartition.Topic, valBytes, 0, valBytes == null ? 0 : valBytes.Length, keyBytes, 0, keyBytes == null ? 0 : keyBytes.Length, message.Timestamp, topicPartition.Partition, headers, handler); return(await handler.Task.ConfigureAwait(false)); } else { ProduceImpl( topicPartition.Topic, valBytes, 0, valBytes == null ? 0 : valBytes.Length, keyBytes, 0, keyBytes == null ? 0 : keyBytes.Length, message.Timestamp, topicPartition.Partition, headers, null); var result = new DeliveryResult <TKey, TValue> { TopicPartitionOffset = new TopicPartitionOffset(topicPartition, Offset.Unset), Message = message }; return(result); } } catch (KafkaException ex) { throw new ProduceException <TKey, TValue>( ex.Error, new DeliveryResult <TKey, TValue> { Message = message, TopicPartitionOffset = new TopicPartitionOffset(topicPartition, Offset.Unset) }); } }