public virtual async Task ReaderSourceTaggedTest()
{
var topic = $"reader-topics-{Guid.NewGuid()}";
var ids = PublishMessages(topic, 50);
var start = MessageIdUtils.GetOffset(ids.First());
var end = MessageIdUtils.GetOffset(ids.Last());
var conf = new ReaderConfigBuilder <DataOp>().Topic(topic);
var reader = _pulsarSystem.EventSource("public", "default", topic, start, end, "http://127.0.0.1:8080")
.Reader(_clientConfigurationData, conf, AvroSchema <DataOp> .Of(typeof(DataOp)))
.SourceMethod()
.CurrentTaggedEvents(new Messages.Consumer.Tag("twitter", "mestical"));
//let leave some time to wire everything up
await Task.Delay(TimeSpan.FromSeconds(20));
var receivedCount = 0;
while (receivedCount == 0)
{
await foreach (var response in reader.CurrentEvents(TimeSpan.FromSeconds(5)))
{
receivedCount++;
_output.WriteLine(JsonSerializer.Serialize(response, new JsonSerializerOptions {
WriteIndented = true
}));
}
await Task.Delay(TimeSpan.FromSeconds(1));
}
Assert.True(receivedCount > 0);
}
private void TestReadMessages(string topic, bool enableBatch)
{
var numKeys = 10;
var builder = new ReaderConfigBuilder <byte[]>()
.Topic(topic)
.StartMessageId(IMessageId.Earliest)
.ReaderName(Subscription);
var reader = _client.NewReader(builder);
var keys = PublishMessages(topic, numKeys, enableBatch);
Thread.Sleep(TimeSpan.FromSeconds(5));
for (var i = 0; i < numKeys; i++)
{
var message = (Message <byte[]>)reader.ReadNext();
if (message != null)
{
_output.WriteLine($"{message.Key}:{message.MessageId}:{Encoding.UTF8.GetString(message.Data)}");
Assert.True(keys.Remove(message.Key));
}
else
{
break;
}
}
Assert.True(keys.Count == 0);
}
public void TestMultiTopic()
{
var topic = "persistent://public/default/topic" + Guid.NewGuid();
var topic2 = "persistent://public/default/topic2" + Guid.NewGuid();
var topic3 = "persistent://public/default/topic3" + Guid.NewGuid();
IList <string> topics = new List <string> {
topic, topic2, topic3
};
var builder = new ReaderConfigBuilder <string>()
.Topics(topics)
.StartMessageId(IMessageId.Earliest)
.ReaderName("my-reader");
var reader = _client.NewReader(ISchema <object> .String, builder);
// create producer and send msg
IList <Producer <string> > producerList = new List <Producer <string> >();
foreach (var topicName in topics)
{
var producer = _client.NewProducer(ISchema <object> .String, new ProducerConfigBuilder <string>().Topic(topicName));
producerList.Add(producer);
}
var msgNum = 10;
ISet <string> messages = new HashSet <string>();
for (var i = 0; i < producerList.Count; i++)
{
var producer = producerList[i];
for (var j = 0; j < msgNum; j++)
{
var msg = i + "msg" + j;
producer.Send(msg);
messages.Add(msg);
}
}
// receive messagesS
var message = reader.ReadNext(TimeSpan.FromSeconds(5));
while (message != null)
{
var value = message.Value;
_output.WriteLine(value);
Assert.True(messages.Remove(value));
message = reader.ReadNext(TimeSpan.FromSeconds(5));
}
Assert.Equal(0, messages.Count);
// clean up
foreach (var producer in producerList)
{
producer.Close();
}
reader.Close();
}
public virtual void ReaderInstantiation()
{
var reader = new ReaderConfigBuilder <string>();
reader.Topic("ReaderInstantiation");
reader.StartMessageId(IMessageId.Earliest);
var stringReaderBuilder = _client.NewReader(new StringSchema(), reader);
Assert.NotNull(stringReaderBuilder);
stringReaderBuilder.Close();
}
public ReaderSourceBuilder(ClientConfigurationData clientConfiguration, ISchema <T> schema, ActorSystem actorSystem, IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef generator, string tenant, string @namespace, string topic, long fromMessageId, long toMessageId, string brokerWebServiceUrl, ReaderConfigBuilder <T> readerConfigBuilder)
{
_clientConfiguration = clientConfiguration;
_schema = schema;
_client = client;
_lookup = lookup;
_cnxPool = cnxPool;
_generator = generator;
_actorSystem = actorSystem;
_fromMessageId = fromMessageId;
_toMessageId = toMessageId;
_tenant = tenant;
_namespace = @namespace;
_topic = topic;
_brokerWebServiceUrl = brokerWebServiceUrl;
_conf = readerConfigBuilder;
}
public virtual void TestReadFromPartition()
{
var topic = "testReadFromPartition";
var partition0 = topic + "-partition-0";
var numKeys = 10;
var builder = new ReaderConfigBuilder <byte[]>()
.Topic(partition0)
.StartMessageId(IMessageId.Earliest)
.ReaderName(Subscription);
var reader = _client.NewReader(builder);
var keys = PublishMessages(partition0, numKeys, false);
Thread.Sleep(TimeSpan.FromSeconds(10));
for (var i = 0; i < numKeys; i++)
{
var message = reader.ReadNext();
Assert.True(keys.Remove(message.Key));
}
Assert.True(keys.Count == 0);
}
public async ValueTask <Reader <T> > NewReaderAsync <T>(ISchema <T> schema, ReaderConfigBuilder <T> confBuilder)
{
var state = await _client.Ask <int>(GetClientState.Instance).ConfigureAwait(false);
if (state != 0)
{
throw new PulsarClientException.AlreadyClosedException("Client already closed");
}
if (confBuilder == null)
{
throw new PulsarClientException.InvalidConfigurationException("Consumer configuration undefined");
}
var conf = confBuilder.ReaderConfigurationData;
foreach (var topic in conf.TopicNames)
{
if (!TopicName.IsValid(topic))
{
throw new PulsarClientException.InvalidTopicNameException("Invalid topic name: '" + topic + "'");
}
}
if (conf.StartMessageId == null)
{
throw new PulsarClientException.InvalidConfigurationException("Invalid startMessageId");
}
if (conf.TopicNames.Count == 1)
{
var schemaClone = await PreProcessSchemaBeforeSubscribe(schema, conf.TopicName).ConfigureAwait(false);
return(await CreateSingleTopicReader(conf, schemaClone).ConfigureAwait(false));
}
return(await CreateMultiTopicReader(conf, schema).ConfigureAwait(false));
}
public ReaderSourceBuilder <T> Reader <T>(ClientConfigurationData clientConfiguration, ReaderConfigBuilder <T> readerConfigBuilder, ISchema <T> schema)
{
if (schema == null)
{
throw new NullReferenceException(nameof(schema));
}
if (readerConfigBuilder == null)
{
throw new NullReferenceException(nameof(readerConfigBuilder));
}
return(new ReaderSourceBuilder <T>(clientConfiguration, schema, _actorSystem, _client, _lookup, _cnxPool, _generator, _tenant, _namespace, _topic, _fromMessageId, _toMessageId, _brokerWebServiceUrl, readerConfigBuilder));
}
public Reader <T> NewReader <T>(ISchema <T> schema, ReaderConfigBuilder <T> confBuilder)
{
return(NewReaderAsync(schema, confBuilder).GetAwaiter().GetResult());
}
public async ValueTask <Reader <byte[]> > NewReaderAsync(ReaderConfigBuilder <byte[]> conf)
{
return(await NewReaderAsync(ISchema <object> .Bytes, conf).ConfigureAwait(false));
}
public Reader <byte[]> NewReader(ReaderConfigBuilder <byte[]> conf)
{
return(NewReaderAsync(conf).GetAwaiter().GetResult());
}
