private void FirstQuery(bool isLive) { try { var max = _message.ToMessageId - _message.FromMessageId; var start = (MessageId)MessageIdUtils.GetMessageId(_message.FromMessageId); var end = (MessageId)MessageIdUtils.GetMessageId(_message.ToMessageId); var query = $"select {string.Join(", ", _message.Columns)},__message_id__, __publish_time__, __properties__, __key__, __producer_name__, __sequence_id__, __partition__ from \"{_message.Topic}\" where __partition__ = {start.PartitionIndex} AND CAST(split_part(replace(replace(__message_id__, '('), ')'), ',', 1) AS BIGINT) BETWEEN bigint '{start.LedgerId}' AND bigint '{end.LedgerId}' AND CAST(split_part(replace(replace(__message_id__, '('), ')'), ',', 2) AS BIGINT) BETWEEN bigint '{start.EntryId}' AND bigint '{end.EntryId}' ORDER BY __publish_time__ ASC LIMIT {max}"; var options = _message.Options; options.Catalog = "pulsar"; options.Schema = "" + _message.Tenant + "/" + _message.Namespace + ""; options.Execute = query; var session = options.ToClientSession(); var executor = new Executor(session, options, _self, _log); _log.Info($"Executing: {options.Execute}"); _ = executor.Run().GetAwaiter().GetResult(); } catch (Exception ex) { _log.Error(ex.ToString()); } if (isLive) { LiveConsume(); } else { Consume(); } }
private void Consume() { Receive <ReceivedMessage <T> >(m => { var c = m.Message; var messageId = MessageIdUtils.GetOffset(m.Message.MessageId); if (messageId <= _toOffset) { var props = c.Properties; var tagged = props.FirstOrDefault(x => x.Key.Equals(_tag.Key, StringComparison.OrdinalIgnoreCase) && x.Value.Equals(_tag.Value, StringComparison.OrdinalIgnoreCase)); if (!string.IsNullOrWhiteSpace(tagged.Value)) { Context.Parent.Tell(m); } //_sequenceId++; } else { Self.GracefulStop(TimeSpan.FromSeconds(5)); } }); Receive <ReceiveTimeout>(t => { Self.GracefulStop(TimeSpan.FromSeconds(5)); }); //to track last sequence id for lagging player Context.SetReceiveTimeout(TimeSpan.FromSeconds(30)); }
private void Query() { try { var max = _currentOffset - _lastEventMessageId; if (max > 0) { var start = (MessageId)MessageIdUtils.GetMessageId(_lastEventMessageId); var end = (MessageId)MessageIdUtils.GetMessageId(_currentOffset); var query = $"select {string.Join(", ", _message.Columns)}, __message_id__, __publish_time__, __properties__, __key__, __producer_name__, __sequence_id__, __partition__ from \"{_message.Topic}\" where __partition__ = {_partitionIndex} AND CAST(split_part(replace(replace(__message_id__, '('), ')'), ',', 1) AS BIGINT) BETWEEN bigint '{start.LedgerId}' AND bigint '{end.LedgerId}' AND CAST(split_part(replace(replace(__message_id__, '('), ')'), ',', 2) AS BIGINT) BETWEEN bigint '{start.EntryId + 1}' AND bigint '{end.EntryId}' ORDER BY __publish_time__ ASC LIMIT {_queryRange}"; var options = _message.Options; options.Catalog = "pulsar"; options.Schema = "" + _message.Tenant + "/" + _message.Namespace + ""; options.Execute = query; var session = options.ToClientSession(); var executor = new Executor(session, options, _self, _log); _log.Info($"Executing: {options.Execute}"); _ = executor.Run().GetAwaiter().GetResult(); _lastEventMessageId = _currentOffset; } } catch (Exception ex) { _log.Error(ex.ToString()); } finally { _queryCancelable = _scheduler.ScheduleOnceCancelable(TimeSpan.FromSeconds(10), Query); } }
private void LiveConsume() { Receive <DataResponse>(c => { for (var i = 0; i < c.Data.Count; i++) { var msgData = c.Data.ElementAt(i); var msg = msgData["__message_id__"].ToString().Trim('(', ')').Split(',').Select(int.Parse).ToArray(); var messageId = MessageIdUtils.GetOffset(new MessageId(msg[0], msg[1], msg[2])); var eventMessage = new EventEnvelope(msgData, messageId, _topicName.ToString()); _buffer.Post(eventMessage); _currentOffset = messageId; } _buffer.Post(new EventStats(new StatsResponse(c.StatementStats))); }); Receive <StatsResponse>(s => { var stats = new EventStats(s); _buffer.Post(stats); }); Receive <ErrorResponse>(s => { var error = new EventError(s); _buffer.Post(error); }); _queryCancelable = _scheduler.ScheduleOnceCancelable(TimeSpan.FromSeconds(60), Query); }
private void Consume() { Receive<DataResponse>(c => { for (var i = 0; i < c.Data.Count; i++) { var msgData = c.Data.ElementAt(i); var msg = msgData["__message_id__"].ToString().Trim('(', ')').Split(',').Select(int.Parse).ToArray(); var messageId = MessageIdUtils.GetOffset(new MessageId(msg[0], msg[1], msg[2])); if (messageId <= _toMessageId) { var eventMessage = new EventEnvelope(msgData, messageId, _topicName.ToString()); _buffer.Post(eventMessage); _currentOffset = messageId; } else Self.GracefulStop(TimeSpan.FromSeconds(5)); } _buffer.Post(new EventStats(new StatsResponse(c.StatementStats))); }); Receive<StatsResponse>(s => { var stats = new EventStats(s); _buffer.Post(stats); }); Receive<ErrorResponse>(s => { var error = new EventError(s); _buffer.Post(error); }); Receive<ReceiveTimeout>(t => { Self.GracefulStop(TimeSpan.FromSeconds(5)); }); //to track last sequence id for lagging player Context.SetReceiveTimeout(TimeSpan.FromSeconds(30)); }
public virtual async Task ReaderSourceTaggedTest() { var topic = $"reader-topics-{Guid.NewGuid()}"; var ids = PublishMessages(topic, 50); var start = MessageIdUtils.GetOffset(ids.First()); var end = MessageIdUtils.GetOffset(ids.Last()); var conf = new ReaderConfigBuilder <DataOp>().Topic(topic); var reader = _pulsarSystem.EventSource("public", "default", topic, start, end, "http://127.0.0.1:8080") .Reader(_clientConfigurationData, conf, AvroSchema <DataOp> .Of(typeof(DataOp))) .SourceMethod() .CurrentTaggedEvents(new Messages.Consumer.Tag("twitter", "mestical")); //let leave some time to wire everything up await Task.Delay(TimeSpan.FromSeconds(20)); var receivedCount = 0; while (receivedCount == 0) { await foreach (var response in reader.CurrentEvents(TimeSpan.FromSeconds(5))) { receivedCount++; _output.WriteLine(JsonSerializer.Serialize(response, new JsonSerializerOptions { WriteIndented = true })); } await Task.Delay(TimeSpan.FromSeconds(1)); } Assert.True(receivedCount > 0); }
public void TestId(long ledger, long entry) { var id = new MessageId(ledger, entry, -1); var offset = MessageIdUtils.GetOffset(id); var id1 = MessageIdUtils.GetMessageId(offset); Assert.Equal(id, id1); }
private void LiveConsume() { Receive <ReceivedMessage <T> >(c => { _parent.Tell(c); _currentOffset = MessageIdUtils.GetOffset(c.Message.MessageId); }); _flowSenderCancelable = _scheduler.ScheduleOnceCancelable(TimeSpan.FromSeconds(60), SendFlow); }
//[Fact] public virtual void SqlSourceTaggedTest() { var topic = $"presto-topics-{Guid.NewGuid()}"; var ids = PublishMessages(topic, 50); var start = MessageIdUtils.GetOffset(ids.First()); var end = MessageIdUtils.GetOffset(ids.Last()); var cols = new HashSet <string> { "text", "EventTime" }; var option = new ClientOptions { Server = "http://127.0.0.1:8081" }; var source = _pulsarSystem.EventSource("public", "default", topic, start, end, "http://127.0.0.1:8080") .Sql(option, cols) .SourceMethod() .CurrentTaggedEvents(new Messages.Consumer.Tag("twitter", "mestical")); var receivedCount = 0; for (var i = 0; i < 45; i++) { var response = source.CurrentEvents(); foreach (var data in response) { i++; receivedCount++; switch (data) { case EventEnvelope dr: _output.WriteLine(JsonSerializer.Serialize(dr, new JsonSerializerOptions { WriteIndented = true })); break; case EventStats sr: _output.WriteLine(JsonSerializer.Serialize(sr.Stats, new JsonSerializerOptions { WriteIndented = true })); break; case EventError er: _output.WriteLine(JsonSerializer.Serialize(er.Error, new JsonSerializerOptions { WriteIndented = true })); break; } } if (receivedCount == 0) { Thread.Sleep(TimeSpan.FromSeconds(10)); } } Assert.True(receivedCount > 0); }
public PrestoSourceActor(BufferBlock <IEventEnvelope> buffer, bool isLive, IPrestoEventSourceMessage message) { _buffer = buffer; _self = Self; _log = Context.GetLogger(); _scheduler = Context.System.Scheduler.Advanced; _topicName = TopicName.Get(message.Topic); _message = message; _toMessageId = message.ToMessageId; _lastEventMessageId = _message.FromMessageId; _partitionIndex = ((MessageId)MessageIdUtils.GetMessageId(message.FromMessageId)).PartitionIndex; _queryRange = message.ToMessageId - message.FromMessageId; FirstQuery(isLive); }
private void LiveConsume() { Receive <ReceivedMessage <T> >(c => { var props = c.Message.Properties; var tagged = props.FirstOrDefault(x => x.Key.Equals(_tag.Key, StringComparison.OrdinalIgnoreCase) && x.Value.Equals(_tag.Value, StringComparison.OrdinalIgnoreCase)); if (!string.IsNullOrWhiteSpace(tagged.Value)) { Context.Parent.Tell(c); } _currentOffset = MessageIdUtils.GetOffset(c.Message.MessageId); //_sequenceId++; }); _flowSenderCancelable = _scheduler.ScheduleOnceCancelable(TimeSpan.FromSeconds(60), SendFlow); }
private void Consume() { Receive <ReceivedMessage <T> >(m => { var c = m.Message; var messageId = MessageIdUtils.GetOffset(m.Message.MessageId); if (messageId <= _toOffset) { _parent.Tell(m); _child.Tell(new AcknowledgeMessage <T>(c)); _child.Tell(new MessageProcessed <T>(c)); //_sequenceId++; } else { Self.GracefulStop(TimeSpan.FromSeconds(5)); } }); Receive <ReceiveTimeout>(t => { Self.GracefulStop(TimeSpan.FromSeconds(5)); }); //to track last sequence id for lagging player Context.SetReceiveTimeout(TimeSpan.FromSeconds(30)); }
private MessageId GetMessageId() { return((MessageId)MessageIdUtils.GetMessageId(_message.FromMessageId)); }