public virtual async Task ReaderSourceTaggedTest() { var topic = $"reader-topics-{Guid.NewGuid()}"; var ids = PublishMessages(topic, 50); var start = MessageIdUtils.GetOffset(ids.First()); var end = MessageIdUtils.GetOffset(ids.Last()); var conf = new ReaderConfigBuilder <DataOp>().Topic(topic); var reader = _pulsarSystem.EventSource("public", "default", topic, start, end, "http://127.0.0.1:8080") .Reader(_clientConfigurationData, conf, AvroSchema <DataOp> .Of(typeof(DataOp))) .SourceMethod() .CurrentTaggedEvents(new Messages.Consumer.Tag("twitter", "mestical")); //let leave some time to wire everything up await Task.Delay(TimeSpan.FromSeconds(20)); var receivedCount = 0; while (receivedCount == 0) { await foreach (var response in reader.CurrentEvents(TimeSpan.FromSeconds(5))) { receivedCount++; _output.WriteLine(JsonSerializer.Serialize(response, new JsonSerializerOptions { WriteIndented = true })); } await Task.Delay(TimeSpan.FromSeconds(1)); } Assert.True(receivedCount > 0); }
private void TestReadMessages(string topic, bool enableBatch) { var numKeys = 10; var builder = new ReaderConfigBuilder <byte[]>() .Topic(topic) .StartMessageId(IMessageId.Earliest) .ReaderName(Subscription); var reader = _client.NewReader(builder); var keys = PublishMessages(topic, numKeys, enableBatch); Thread.Sleep(TimeSpan.FromSeconds(5)); for (var i = 0; i < numKeys; i++) { var message = (Message <byte[]>)reader.ReadNext(); if (message != null) { _output.WriteLine($"{message.Key}:{message.MessageId}:{Encoding.UTF8.GetString(message.Data)}"); Assert.True(keys.Remove(message.Key)); } else { break; } } Assert.True(keys.Count == 0); }
public void TestMultiTopic() { var topic = "persistent://public/default/topic" + Guid.NewGuid(); var topic2 = "persistent://public/default/topic2" + Guid.NewGuid(); var topic3 = "persistent://public/default/topic3" + Guid.NewGuid(); IList <string> topics = new List <string> { topic, topic2, topic3 }; var builder = new ReaderConfigBuilder <string>() .Topics(topics) .StartMessageId(IMessageId.Earliest) .ReaderName("my-reader"); var reader = _client.NewReader(ISchema <object> .String, builder); // create producer and send msg IList <Producer <string> > producerList = new List <Producer <string> >(); foreach (var topicName in topics) { var producer = _client.NewProducer(ISchema <object> .String, new ProducerConfigBuilder <string>().Topic(topicName)); producerList.Add(producer); } var msgNum = 10; ISet <string> messages = new HashSet <string>(); for (var i = 0; i < producerList.Count; i++) { var producer = producerList[i]; for (var j = 0; j < msgNum; j++) { var msg = i + "msg" + j; producer.Send(msg); messages.Add(msg); } } // receive messagesS var message = reader.ReadNext(TimeSpan.FromSeconds(5)); while (message != null) { var value = message.Value; _output.WriteLine(value); Assert.True(messages.Remove(value)); message = reader.ReadNext(TimeSpan.FromSeconds(5)); } Assert.Equal(0, messages.Count); // clean up foreach (var producer in producerList) { producer.Close(); } reader.Close(); }
public virtual void ReaderInstantiation() { var reader = new ReaderConfigBuilder <string>(); reader.Topic("ReaderInstantiation"); reader.StartMessageId(IMessageId.Earliest); var stringReaderBuilder = _client.NewReader(new StringSchema(), reader); Assert.NotNull(stringReaderBuilder); stringReaderBuilder.Close(); }
public ReaderSourceBuilder(ClientConfigurationData clientConfiguration, ISchema <T> schema, ActorSystem actorSystem, IActorRef client, IActorRef lookup, IActorRef cnxPool, IActorRef generator, string tenant, string @namespace, string topic, long fromMessageId, long toMessageId, string brokerWebServiceUrl, ReaderConfigBuilder <T> readerConfigBuilder) { _clientConfiguration = clientConfiguration; _schema = schema; _client = client; _lookup = lookup; _cnxPool = cnxPool; _generator = generator; _actorSystem = actorSystem; _fromMessageId = fromMessageId; _toMessageId = toMessageId; _tenant = tenant; _namespace = @namespace; _topic = topic; _brokerWebServiceUrl = brokerWebServiceUrl; _conf = readerConfigBuilder; }
public virtual void TestReadFromPartition() { var topic = "testReadFromPartition"; var partition0 = topic + "-partition-0"; var numKeys = 10; var builder = new ReaderConfigBuilder <byte[]>() .Topic(partition0) .StartMessageId(IMessageId.Earliest) .ReaderName(Subscription); var reader = _client.NewReader(builder); var keys = PublishMessages(partition0, numKeys, false); Thread.Sleep(TimeSpan.FromSeconds(10)); for (var i = 0; i < numKeys; i++) { var message = reader.ReadNext(); Assert.True(keys.Remove(message.Key)); } Assert.True(keys.Count == 0); }
public async ValueTask <Reader <T> > NewReaderAsync <T>(ISchema <T> schema, ReaderConfigBuilder <T> confBuilder) { var state = await _client.Ask <int>(GetClientState.Instance).ConfigureAwait(false); if (state != 0) { throw new PulsarClientException.AlreadyClosedException("Client already closed"); } if (confBuilder == null) { throw new PulsarClientException.InvalidConfigurationException("Consumer configuration undefined"); } var conf = confBuilder.ReaderConfigurationData; foreach (var topic in conf.TopicNames) { if (!TopicName.IsValid(topic)) { throw new PulsarClientException.InvalidTopicNameException("Invalid topic name: '" + topic + "'"); } } if (conf.StartMessageId == null) { throw new PulsarClientException.InvalidConfigurationException("Invalid startMessageId"); } if (conf.TopicNames.Count == 1) { var schemaClone = await PreProcessSchemaBeforeSubscribe(schema, conf.TopicName).ConfigureAwait(false); return(await CreateSingleTopicReader(conf, schemaClone).ConfigureAwait(false)); } return(await CreateMultiTopicReader(conf, schema).ConfigureAwait(false)); }
public ReaderSourceBuilder <T> Reader <T>(ClientConfigurationData clientConfiguration, ReaderConfigBuilder <T> readerConfigBuilder, ISchema <T> schema) { if (schema == null) { throw new NullReferenceException(nameof(schema)); } if (readerConfigBuilder == null) { throw new NullReferenceException(nameof(readerConfigBuilder)); } return(new ReaderSourceBuilder <T>(clientConfiguration, schema, _actorSystem, _client, _lookup, _cnxPool, _generator, _tenant, _namespace, _topic, _fromMessageId, _toMessageId, _brokerWebServiceUrl, readerConfigBuilder)); }
public Reader <T> NewReader <T>(ISchema <T> schema, ReaderConfigBuilder <T> confBuilder) { return(NewReaderAsync(schema, confBuilder).GetAwaiter().GetResult()); }
public async ValueTask <Reader <byte[]> > NewReaderAsync(ReaderConfigBuilder <byte[]> conf) { return(await NewReaderAsync(ISchema <object> .Bytes, conf).ConfigureAwait(false)); }
public Reader <byte[]> NewReader(ReaderConfigBuilder <byte[]> conf) { return(NewReaderAsync(conf).GetAwaiter().GetResult()); }