public override Task ConsumeAsync(ConsumedMessage <Request> message)
{
return(Task.Factory
.StartNew(() =>
{
Console.WriteLine("[{0}]received '{1}' by '{2}' with correlation id {3}",
message.ConsumerTag,
message.Headers.MessageId,
GetType().Name,
message.Headers.CorrelationId);
var exchange = _broker.DeclareDirectExchange(message.Headers
.ReplyConfiguration
.ExchangeName);
var queue = _broker.DeclareQueue(message.Headers
.ReplyConfiguration
.RoutingKey);
_broker.DeclareExchangeBinding(exchange,
queue,
message.Headers.ReplyConfiguration.RoutingKey);
var outboundMessage = new OutboundMessage <Response>(new Response
{
BarBar = message.Content.Bar * 2
});
outboundMessage.SetCorrelationId(message.Headers.CorrelationId);
_connection.PublishAsync(outboundMessage,
exchange,
message.Headers.ReplyConfiguration.RoutingKey);
}));
}
public async Task ThenItShouldUpsertProviderWhenMessageReceived()
{
var cancellationToken = new CancellationToken();
var message = new ConsumedMessage <string, Provider>
{
Topic = "topic-name",
Partition = 91,
Offset = 28379,
Key = "12345678",
Value = new Provider(),
};
var expectedFeProvider = new FeProvider();
_mapperMock.Setup(m => m.Map <FeProvider>(It.IsAny <Provider>()))
.Returns(expectedFeProvider);
Func <ConsumedMessage <string, Provider>, CancellationToken, Task> messageHandler = null;
_ukrlpConsumerMock.Setup(c => c.SetMessageHandler(It.IsAny <Func <ConsumedMessage <string, Provider>, CancellationToken, Task> >()))
.Callback((Func <ConsumedMessage <string, Provider>, CancellationToken, Task> handler) => { messageHandler = handler; });
_worker = new Worker(
_ukrlpConsumerMock.Object,
_providerManagerMock.Object,
_options,
_mapperMock.Object,
_loggerMock.Object);
var workerTask = _worker.StartAsync(CancellationToken.None);
await messageHandler.Invoke(message, cancellationToken);
await workerTask;
_providerManagerMock.Verify(m => m.UpsertProvider(expectedFeProvider, cancellationToken));
}
public async Task ThenItShouldUpdateLearnersLearningDeliveriesAfterSettingADeliverIdAndLearnerId()
{
var cancellationToken = new CancellationToken();
var message = new ConsumedMessage <string, Sld.Learner>
{
Topic = "topic-name",
Partition = 91,
Offset = 28379,
Key = "12345678-123",
Value = new Sld.Learner
{
LearningDeliveries = new Sld.LearningDelivery[3]
},
};
var learnerId = Guid.NewGuid();
var expectedLearningDeliveries = Enumerable.Range(1, 3).Select(x => new Domain.LearningDeliveries.LearningDelivery()).ToArray();
_learnerManagerMock.Setup(m => m.UpsertLearner(It.IsAny <Domain.Learners.Learner>(), It.IsAny <CancellationToken>()))
.ReturnsAsync(learnerId);
_mapperMock.Setup(m => m.Map <Domain.LearningDeliveries.LearningDelivery[]>(It.IsAny <Sld.LearningDelivery[]>()))
.Returns(expectedLearningDeliveries);
Func <ConsumedMessage <string, Sld.Learner>, CancellationToken, Task> messageHandler = null;
_sldConsumerMock.Setup(c => c.SetMessageHandler(It.IsAny <Func <ConsumedMessage <string, Sld.Learner>, CancellationToken, Task> >()))
.Callback((Func <ConsumedMessage <string, Sld.Learner>, CancellationToken, Task> handler) => { messageHandler = handler; });
_worker = new Worker(
_sldConsumerMock.Object,
_learnerManagerMock.Object,
_learningDeliveryManagerMock.Object,
_options,
_mapperMock.Object,
_loggerMock.Object);
var workerTask = _worker.StartAsync(CancellationToken.None);
await messageHandler.Invoke(message, cancellationToken);
await workerTask;
_learningDeliveryManagerMock.Verify(m => m.UpdateLearnersLearningDeliveriesAsync(
It.IsAny <Domain.LearningDeliveries.LearningDelivery[]>(),
It.IsAny <CancellationToken>()),
Times.Once);
_learningDeliveryManagerMock.Verify(m => m.UpdateLearnersLearningDeliveriesAsync(
It.Is <Domain.LearningDeliveries.LearningDelivery[]>(ld => ld.Length == 3),
cancellationToken),
Times.Once);
_learningDeliveryManagerMock.Verify(m => m.UpdateLearnersLearningDeliveriesAsync(
It.Is <Domain.LearningDeliveries.LearningDelivery[]>(ld => ld[0].LearnerId == learnerId && ld[0].Id != Guid.Empty),
cancellationToken),
Times.Once);
_learningDeliveryManagerMock.Verify(m => m.UpdateLearnersLearningDeliveriesAsync(
It.Is <Domain.LearningDeliveries.LearningDelivery[]>(ld => ld[1].LearnerId == learnerId && ld[1].Id != Guid.Empty),
cancellationToken),
Times.Once);
_learningDeliveryManagerMock.Verify(m => m.UpdateLearnersLearningDeliveriesAsync(
It.Is <Domain.LearningDeliveries.LearningDelivery[]>(ld => ld[2].LearnerId == learnerId && ld[2].Id != Guid.Empty),
cancellationToken),
Times.Once);
}
public void TestTlsLargeSizeMessage()
{
_output.WriteLine($"-- Starting 'TestTlsLargeSizeMessage' test --");
const int messageSize = 16 * 1024 + 1;
InternalSetUpForClient(true, "pulsar.BrokerServiceUrlTls");
InternalSetUpForNamespace();
var consumer = _common.PulsarSystem.PulsarConsumer(_common.CreateConsumer(BytesSchema.Of(), "persistent://my-property/use/my-ns/my-topic1", "TestTlsLargeSizeMessage", "my-subscriber-name"));
var producer = _common.PulsarSystem.PulsarProducer(_common.CreateProducer(BytesSchema.Of(), "persistent://my-property/use/my-ns/my-topic1", "TestTlsLargeSizeMessage"));
for (int i = 0; i < 10; i++)
{
var message = new byte[messageSize];
Array.Fill(message, (byte)i);
var send = new Send(message, ImmutableDictionary <string, object> .Empty);
_common.PulsarSystem.Send(send, producer.Producer);
}
ConsumedMessage msg = null;
for (int i = 0; i < 10; i++)
{
msg = _common.PulsarSystem.Receive("", 5000);
var expected = new byte[messageSize];
Array.Fill(expected, (byte)i);
var data = (byte[])(object)msg.Message.Data;
Assert.Equal(expected, data);
}
// Acknowledge the consumption of all messages at once
_common.PulsarSystem.AcknowledgeCumulative(msg);
_output.WriteLine($"-- Exiting 'TestTlsLargeSizeMessage' test --");
}
/// <inheritdoc />
public async Task <PullResult> PullAsync(CancellationToken cancellationToken = default)
{
using var cts = cancellationToken.WithTimeout(options.Timeout);
var basicGetResult = await channel.InvokeChannelActionAsync(
x => x.BasicGet(queue.Name, options.AutoAck), cts.Token
).ConfigureAwait(false);
if (basicGetResult == null)
{
return(PullResult.NotAvailable);
}
var messagesCount = basicGetResult.MessageCount;
var message = new ConsumedMessage(
new MessageReceivedInfo(
"",
basicGetResult.DeliveryTag,
basicGetResult.Redelivered,
basicGetResult.Exchange,
basicGetResult.RoutingKey,
queue.Name
),
new MessageProperties(basicGetResult.BasicProperties),
basicGetResult.Body.ToArray()
);
var interceptedMessage = interceptor.OnConsume(message);
return(PullResult.Available(
messagesCount, interceptedMessage.ReceivedInfo, interceptedMessage.Properties, interceptedMessage.Body
));
}
public void ConsumingSuccesfully()
{
var consumer = new FakeConsumer(_ => Task.Factory.StartNew(() => { }));
var result = new ConsumedMessage(new Foo(),
FakeBasicDeliverEventArgs()).ConsumeAsync(new[] { consumer })
.Result;
Assert.IsType <Success>(result);
}
public override Task ConsumeAsync(ConsumedMessage <Foo> message)
{
return(Task.Factory.StartNew(() =>
{
Console.WriteLine("[{0}]received '{1}' by '{2}'",
message.ConsumerTag,
message.Headers.MessageId,
GetType().Name);
}));
}
public void ShouldCompressAndDecompress()
{
var interceptor = new GZipInterceptor();
var body = Encoding.UTF8.GetBytes("haha");
var outgoingMessage = new ProducedMessage(new MessageProperties(), body);
var message = interceptor.OnProduce(outgoingMessage);
var incomingMessage = new ConsumedMessage(null, message.Properties, message.Body);
Assert.Equal(body, interceptor.OnConsume(incomingMessage).Body);
}
public void ShouldEncryptAndDecrypt()
{
var interceptor = new TripleDESInterceptor(Convert.FromBase64String("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), Convert.FromBase64String("aaaaaaaaaaa="));
var body = Encoding.UTF8.GetBytes("haha");
var outgoingMessage = new ProducedMessage(new MessageProperties(), body);
var message = interceptor.OnProduce(outgoingMessage);
var incomingMessage = new ConsumedMessage(null, message.Properties, message.Body);
Assert.Equal(body, interceptor.OnConsume(incomingMessage).Body);
}
public override Task ConsumeAsync(ConsumedMessage <Response> message)
{
return(Task.Factory
.StartNew(() =>
{
Console.WriteLine("[{0}]received '{1}' by '{2}' with correlation id {3}",
message.ConsumerTag,
message.Headers.MessageId,
GetType().Name,
message.Headers.CorrelationId);
}));
}
/// <inheritdoc />
public ConsumedMessage OnConsume(ConsumedMessage message)
{
var receivedInfo = message.ReceivedInfo;
var properties = message.Properties;
var body = message.Body;
using var output = new MemoryStream();
using (var compressedStream = new MemoryStream(body))
using (var decompressingStream = new GZipStream(compressedStream, CompressionMode.Decompress))
decompressingStream.CopyTo(output);
return(new ConsumedMessage(receivedInfo, properties, output.ToArray()));
}
/// <inheritdoc />
public ConsumedMessage OnConsume(ConsumedMessage message)
{
using var tripleDes = TripleDES.Create();
using var tripleDesDecryptor = tripleDes.CreateDecryptor(key, iv);
var receivedInfo = message.ReceivedInfo;
var properties = message.Properties;
var body = message.Body;
return(new ConsumedMessage(
receivedInfo, properties, tripleDesDecryptor.TransformFinalBlock(body, 0, body.Length)
));
}
private async Task ProcessMessageFromTopic(ConsumedMessage <string, Provider> message, CancellationToken cancellationToken)
{
_logger.LogInformation("Received update for provider {UKPRN} - {ProviderName} (topic: {Topic}, partition: {Partition}, offset: {Offset})",
message.Value.UnitedKingdomProviderReferenceNumber,
message.Value.ProviderName,
message.Topic,
message.Partition,
message.Offset);
var feProvider = _mapper.Map <FeProvider>(message.Value);
await _providerManager.UpsertProvider(feProvider, cancellationToken);
}
public void Throws()
{
const String message = "boom";
var exception = new Exception(message);
var consumer = new FakeConsumer(_ => { throw exception; });
var result = new ConsumedMessage(new Foo(),
FakeBasicDeliverEventArgs()).ConsumeAsync(new[] { consumer }).Result;
var actual = Assert.IsType <ConsumingFailure>(result);
Assert.Equal(1, actual.Exceptions.Length);
Assert.Equal(message, actual.Exceptions.First().Message);
Assert.Equal(1, consumer.Errors.Count);
Assert.Equal(message, consumer.Errors.First().Message);
}
public void ThrowsOnRedelivered()
{
const String message = "boom";
var exception = new Exception(message);
var consumer = new FakeConsumer(_ => { throw exception; });
var result = new ConsumedMessage(new Foo(),
new BasicDeliverEventArgs
{
Redelivered = true,
BasicProperties = new BasicProperties()
}).ConsumeAsync(new[] { consumer }).Result;
var actual = Assert.IsType <ReiteratedConsumingFailure>(result);
Assert.Equal(1, actual.Exceptions.Length);
Assert.Equal(message, actual.Exceptions.First().Message);
Assert.Equal(1, consumer.Errors.Count);
Assert.Equal(message, consumer.Errors.First().Message);
}
public void ShouldCallAddedInterceptorsOnConsume()
{
var sourceMessage = new ConsumedMessage(null, new MessageProperties(), new byte[0]);
var firstMessage = new ConsumedMessage(null, new MessageProperties(), new byte[0]);
var secondMessage = new ConsumedMessage(null, new MessageProperties(), new byte[0]);
var first = Substitute.For <IProduceConsumeInterceptor>();
var second = Substitute.For <IProduceConsumeInterceptor>();
first.OnConsume(secondMessage).Returns(firstMessage);
second.OnConsume(sourceMessage).Returns(secondMessage);
var compositeInterceptor = new CompositeInterceptor();
compositeInterceptor.Add(first);
compositeInterceptor.Add(second);
Assert.Equal(firstMessage, compositeInterceptor.OnConsume(sourceMessage));
}
private async Task ProcessMessageFromTopic(ConsumedMessage <string, Sld.Learner> message, CancellationToken cancellationToken)
{
_logger.LogInformation("Received update for learner {LearnRefNumber} at provider {UKPRN} (topic: {Topic}, partition: {Partition}, offset: {Offset})",
message.Value.LearnRefNumber,
message.Value.Ukprn,
message.Topic,
message.Partition,
message.Offset);
var learner = _mapper.Map <Domain.Learners.Learner>(message.Value);
var learnerId = await _learnerManager.UpsertLearner(learner, cancellationToken);
var learningDeliveries = _mapper.Map <Domain.LearningDeliveries.LearningDelivery[]>(message.Value.LearningDeliveries ?? new Sld.LearningDelivery[0]);
foreach (var learningDelivery in learningDeliveries)
{
learningDelivery.Id = Guid.NewGuid();
learningDelivery.LearnerId = learnerId;
}
await _learningDeliveryManager.UpdateLearnersLearningDeliveriesAsync(learningDeliveries, cancellationToken);
}
public void TestTlsCertsFromDynamicStream()
{
string topicName = "persistent://my-property/use/my-ns/my-topic1";
AtomicInteger index = new AtomicInteger(0);
MemoryStream certStream = CreateByteInputStream(TlsClientCertFilePath);
MemoryStream keyStream = CreateByteInputStream(TlsClientKeyFilePath);
Func <MemoryStream> certProvider = () => GetStream(index, certStream);
Func <MemoryStream> keyProvider = () => GetStream(index, keyStream);
AuthenticationTls auth = new AuthenticationTls(certProvider, keyProvider);
_common.GetPulsarSystem(auth, 1000, enableTls: true, brokerService: "pulsar.BrokerServiceUrlTls");
_common.PulsarSystem.PulsarConsumer(_common.CreateConsumer(BytesSchema.Of(), topicName, "TestTlsCertsFromDynamicStream", "my-subscriber-name"));
// unload the topic so, new connection will be made and read the cert streams again
var producer = _common.PulsarSystem.PulsarProducer(_common.CreateProducer(BytesSchema.Of(), "persistent://my-property/use/my-ns/my-topic1", ""));
for (int i = 0; i < 10; i++)
{
var send = new Send(("test" + i).GetBytes(), ImmutableDictionary <string, object> .Empty);
_common.PulsarSystem.Send(send, producer.Producer);
}
ConsumedMessage msg = null;
for (var i = 0; i < 10; i++)
{
msg = _common.PulsarSystem.Receive("TestTlsCertsFromDynamicStream", 5000);
var exepctedMsg = "test" + i;
var data = (byte[])(object)msg.Message.Data;
Assert.Equal(exepctedMsg.GetBytes(), data);
}
// Acknowledge the consumption of all messages at once
_common.PulsarSystem.AcknowledgeCumulative(msg);
}
public async Task ThenItShouldMapLearningDeliveries()
{
var cancellationToken = new CancellationToken();
var message = new ConsumedMessage <string, Sld.Learner>
{
Topic = "topic-name",
Partition = 91,
Offset = 28379,
Key = "12345678-123",
Value = new Sld.Learner
{
LearningDeliveries = new Sld.LearningDelivery[3]
},
};
var expectedLearningDeliveries = Enumerable.Range(1, 3).Select(x => new Domain.LearningDeliveries.LearningDelivery()).ToArray();
_mapperMock.Setup(m => m.Map <Domain.LearningDeliveries.LearningDelivery[]>(It.IsAny <Sld.LearningDelivery[]>()))
.Returns(expectedLearningDeliveries);
Func <ConsumedMessage <string, Sld.Learner>, CancellationToken, Task> messageHandler = null;
_sldConsumerMock.Setup(c => c.SetMessageHandler(It.IsAny <Func <ConsumedMessage <string, Sld.Learner>, CancellationToken, Task> >()))
.Callback((Func <ConsumedMessage <string, Sld.Learner>, CancellationToken, Task> handler) => { messageHandler = handler; });
_worker = new Worker(
_sldConsumerMock.Object,
_learnerManagerMock.Object,
_learningDeliveryManagerMock.Object,
_options,
_mapperMock.Object,
_loggerMock.Object);
var workerTask = _worker.StartAsync(CancellationToken.None);
await messageHandler.Invoke(message, cancellationToken);
await workerTask;
_mapperMock.Verify(m => m.Map <Domain.LearningDeliveries.LearningDelivery[]>(message.Value.LearningDeliveries),
Times.Once);
}
public void then_properties_should_be_set()
{
var message = new KafkaMessage(
"value1".Encode(),
"key1".Encode(),
CompressionCodecs.GZIPCompressionCodec
)
{
PartitionId = 1,
Offset = 1234
};
var actual = message.AsConsumedMessage();
var expected = new ConsumedMessage
{
Codec = Compression.GZip,
Key = "key1",
Value = "value1",
Partition = 1,
Offset = 1234
};
actual.ShouldBeEquivalentTo(expected);
}
public override Task ConsumeAsync(ConsumedMessage <Foo> message)
{
return(Task.FromResult(0));
}
public override Task ConsumeAsync(ConsumedMessage <Foo> message)
{
return(_func(message));
}
static Task MessageReceived(ConsumedMessage <string, string> message, CancellationToken cancellationToken)
{
Console.WriteLine($"[P:{message.Partition}, O:{message.Offset}]: {message.Key} / {message.Value}");
return(Task.CompletedTask);
}
/// <inheritdoc /> public ConsumedMessage OnConsume(ConsumedMessage message) => message;
/// <inheritdoc />
public ConsumedMessage OnConsume(ConsumedMessage message)
{
return(interceptors.AsEnumerable()
.Reverse()
.Aggregate(message, (x, y) => y.OnConsume(x)));
}
public abstract Task ConsumeAsync(ConsumedMessage <TMessage> message);
