public async Task <OperateResult> SendAsync(TransportMessage message) { var producer = _connectionPool.RentProducer(); try { var headers = new Confluent.Kafka.Headers(); foreach (var header in message.Headers) { headers.Add(header.Value != null ? new Header(header.Key, Encoding.UTF8.GetBytes(header.Value)) : new Header(header.Key, null)); } var result = await producer.ProduceAsync(message.GetName(), new Message <string, byte[]> { Headers = headers, Key = message.Headers.TryGetValue(KafkaHeaders.KafkaKey, out string kafkaMessageKey) && !string.IsNullOrEmpty(kafkaMessageKey) ? kafkaMessageKey : message.GetId(), Value = message.Body }); if (result.Status == PersistenceStatus.Persisted || result.Status == PersistenceStatus.PossiblyPersisted) { _logger.LogDebug($"kafka topic message [{message.GetName()}] has been published."); return(OperateResult.Success); } throw new PublisherSentFailedException("kafka message persisted failed!"); }
public async Task <OperateResult> SendAsync(TransportMessage message) { var producer = _connectionPool.RentProducer(); try { var headers = new Confluent.Kafka.Headers(); foreach (var header in message.Headers) { headers.Add(header.Value != null ? new Header(header.Key, Encoding.UTF8.GetBytes(header.Value)) : new Header(header.Key, null)); } var result = await producer.ProduceAsync(message.GetName(), new Message <string, byte[]> { Headers = headers, Key = message.GetId(), Value = message.Body }); if (result.Status == PersistenceStatus.Persisted || result.Status == PersistenceStatus.PossiblyPersisted) { _logger.LogDebug($"kafka topic message [{message.GetName()}] has been published."); return(OperateResult.Success); } var ex = new PublisherSentFailedException("kafka message persisted failed!"); return(OperateResult.Failed(ex)); } catch (Exception ex) { var wapperEx = new PublisherSentFailedException(ex.Message, ex); return(OperateResult.Failed(wapperEx)); } finally { var returned = _connectionPool.Return(producer); if (!returned) { producer.Dispose(); } } }
private static Message <byte[], byte[]> CreateMessage(IMessageContext context) { var headers = new Confluent.Kafka.Headers(); foreach (var header in context.Headers) { headers.Add(header.Value != null ? new Header(header.Key, header.Value) : new Header(header.Key, null)); } return(new Message <byte[], byte[]> { Key = context.PartitionKey, Value = GetMessageContent(context), Headers = headers, Timestamp = Timestamp.Default }); }
public void GetKafkaHeader_ShouldReturnKafkaHeaders() { // Arrange var kafkaHeaders = new Headers { { key, this.value } }; var messageHeaders = new MessageHeaders(); messageHeaders.Add(key, this.value); var result = new Confluent.Kafka.Headers(); foreach (var header in messageHeaders) { result.Add(header.Value != null ? new Header(header.Key, header.Value) : new Header(header.Key, null)); } // Assert result.Should().BeEquivalentTo(kafkaHeaders); }
private ConsumeResult <K, V> ConsumeImpl <K, V>( int millisecondsTimeout, IDeserializer <K> keyDeserializer, IDeserializer <V> valueDeserializer) { var msgPtr = kafkaHandle.ConsumerPoll((IntPtr)millisecondsTimeout); if (msgPtr == IntPtr.Zero) { return(null); } try { var msg = Util.Marshal.PtrToStructure <rd_kafka_message>(msgPtr); string topic = null; if (this.enableTopicNameMarshaling) { if (msg.rkt != IntPtr.Zero) { topic = Util.Marshal.PtrToStringUTF8(Librdkafka.topic_name(msg.rkt)); } } if (msg.err == ErrorCode.Local_PartitionEOF) { return(new ConsumeResult <K, V> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = null, IsPartitionEOF = true }); } long timestampUnix = 0; IntPtr timestampType = (IntPtr)TimestampType.NotAvailable; if (enableTimestampMarshaling) { timestampUnix = Librdkafka.message_timestamp(msgPtr, out timestampType); } var timestamp = new Timestamp(timestampUnix, (TimestampType)timestampType); Headers headers = null; if (enableHeaderMarshaling) { headers = new Headers(); Librdkafka.message_headers(msgPtr, out IntPtr hdrsPtr); if (hdrsPtr != IntPtr.Zero) { for (var i = 0; ; ++i) { var err = Librdkafka.header_get_all(hdrsPtr, (IntPtr)i, out IntPtr namep, out IntPtr valuep, out IntPtr sizep); if (err != ErrorCode.NoError) { break; } var headerName = Util.Marshal.PtrToStringUTF8(namep); byte[] headerValue = null; if (valuep != IntPtr.Zero) { headerValue = new byte[(int)sizep]; Marshal.Copy(valuep, headerValue, 0, (int)sizep); } headers.Add(headerName, headerValue); } } } if (msg.err != ErrorCode.NoError) { throw new ConsumeException( new ConsumeResult <byte[], byte[]> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = new Message <byte[], byte[]> { Timestamp = timestamp, Headers = headers, Key = KeyAsByteArray(msg), Value = ValueAsByteArray(msg) }, IsPartitionEOF = false }, kafkaHandle.CreatePossiblyFatalError(msg.err, null)); } K key; try { unsafe { key = keyDeserializer.Deserialize( msg.key == IntPtr.Zero ? ReadOnlySpan <byte> .Empty : new ReadOnlySpan <byte>(msg.key.ToPointer(), (int)msg.key_len), msg.key == IntPtr.Zero, new SerializationContext(MessageComponentType.Key, topic)); } } catch (Exception ex) { throw new ConsumeException( new ConsumeResult <byte[], byte[]> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = new Message <byte[], byte[]> { Timestamp = timestamp, Headers = headers, Key = KeyAsByteArray(msg), Value = ValueAsByteArray(msg) }, IsPartitionEOF = false }, new Error(ErrorCode.Local_KeyDeserialization), ex); } V val; try { unsafe { val = valueDeserializer.Deserialize( msg.val == IntPtr.Zero ? ReadOnlySpan <byte> .Empty : new ReadOnlySpan <byte>(msg.val.ToPointer(), (int)msg.len), msg.val == IntPtr.Zero, new SerializationContext(MessageComponentType.Value, topic)); } } catch (Exception ex) { throw new ConsumeException( new ConsumeResult <byte[], byte[]> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = new Message <byte[], byte[]> { Timestamp = timestamp, Headers = headers, Key = KeyAsByteArray(msg), Value = ValueAsByteArray(msg) }, IsPartitionEOF = false }, new Error(ErrorCode.Local_ValueDeserialization), ex); } return(new ConsumeResult <K, V> { TopicPartitionOffset = new TopicPartitionOffset(topic, msg.partition, msg.offset), Message = new Message <K, V> { Timestamp = timestamp, Headers = headers, Key = key, Value = val }, IsPartitionEOF = false }); } finally { Librdkafka.message_destroy(msgPtr); } }
private void DeliveryReportCallbackImpl(IntPtr rk, IntPtr rkmessage, IntPtr opaque) { // Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios). if (ownedKafkaHandle.IsClosed) { return; } try { var msg = Util.Marshal.PtrToStructure <rd_kafka_message>(rkmessage); // the msg._private property has dual purpose. Here, it is an opaque pointer set // by Topic.Produce to be an IDeliveryHandler. When Consuming, it's for internal // use (hence the name). if (msg._private == IntPtr.Zero) { // Note: this can occur if the ProduceAsync overload that accepts a DeliveryHandler // was used and the delivery handler was set to null. return; } var gch = GCHandle.FromIntPtr(msg._private); var deliveryHandler = (IDeliveryHandler)gch.Target; gch.Free(); Headers headers = null; if (this.enableDeliveryReportHeaders) { headers = new Headers(); Librdkafka.message_headers(rkmessage, out IntPtr hdrsPtr); if (hdrsPtr != IntPtr.Zero) { for (var i = 0; ; ++i) { var err = Librdkafka.header_get_all(hdrsPtr, (IntPtr)i, out IntPtr namep, out IntPtr valuep, out IntPtr sizep); if (err != ErrorCode.NoError) { break; } var headerName = Util.Marshal.PtrToStringUTF8(namep); byte[] headerValue = null; if (valuep != IntPtr.Zero) { headerValue = new byte[(int)sizep]; Marshal.Copy(valuep, headerValue, 0, (int)sizep); } headers.Add(headerName, headerValue); } } } IntPtr timestampType = (IntPtr)TimestampType.NotAvailable; long timestamp = 0; if (enableDeliveryReportTimestamp) { timestamp = Librdkafka.message_timestamp(rkmessage, out timestampType); } PersistenceStatus messageStatus = PersistenceStatus.PossiblyPersisted; if (enableDeliveryReportPersistedStatus) { messageStatus = Librdkafka.message_status(rkmessage); } deliveryHandler.HandleDeliveryReport( new DeliveryReport <Null, Null> { // Topic is not set here in order to avoid the marshalling cost. // Instead, the delivery handler is expected to cache the topic string. Partition = msg.partition, Offset = msg.offset, Error = KafkaHandle.CreatePossiblyFatalError(msg.err, null), Status = messageStatus, Message = new Message <Null, Null> { Timestamp = new Timestamp(timestamp, (TimestampType)timestampType), Headers = headers } } ); } catch (Exception e) { handlerException = e; } }