private void FirstQuery(bool isLive)
{
try
{
var max = _message.ToMessageId - _message.FromMessageId;
var start = (MessageId)MessageIdUtils.GetMessageId(_message.FromMessageId);
var end = (MessageId)MessageIdUtils.GetMessageId(_message.ToMessageId);
var query = $"select {string.Join(", ", _message.Columns)},__message_id__, __publish_time__, __properties__, __key__, __producer_name__, __sequence_id__, __partition__ from \"{_message.Topic}\" where __partition__ = {start.PartitionIndex} AND CAST(split_part(replace(replace(__message_id__, '('), ')'), ',', 1) AS BIGINT) BETWEEN bigint '{start.LedgerId}' AND bigint '{end.LedgerId}' AND CAST(split_part(replace(replace(__message_id__, '('), ')'), ',', 2) AS BIGINT) BETWEEN bigint '{start.EntryId}' AND bigint '{end.EntryId}' ORDER BY __publish_time__ ASC LIMIT {max}";
var options = _message.Options;
options.Catalog = "pulsar";
options.Schema = "" + _message.Tenant + "/" + _message.Namespace + "";
options.Execute = query;
var session = options.ToClientSession();
var executor = new Executor(session, options, _self, _log);
_log.Info($"Executing: {options.Execute}");
_ = executor.Run().GetAwaiter().GetResult();
}
catch (Exception ex)
{
_log.Error(ex.ToString());
}
if (isLive)
{
LiveConsume();
}
else
{
Consume();
}
}
private void Consume()
{
Receive <ReceivedMessage <T> >(m =>
{
var c = m.Message;
var messageId = MessageIdUtils.GetOffset(m.Message.MessageId);
if (messageId <= _toOffset)
{
var props = c.Properties;
var tagged = props.FirstOrDefault(x => x.Key.Equals(_tag.Key, StringComparison.OrdinalIgnoreCase) && x.Value.Equals(_tag.Value, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(tagged.Value))
{
Context.Parent.Tell(m);
}
//_sequenceId++;
}
else
{
Self.GracefulStop(TimeSpan.FromSeconds(5));
}
});
Receive <ReceiveTimeout>(t => { Self.GracefulStop(TimeSpan.FromSeconds(5)); });
//to track last sequence id for lagging player
Context.SetReceiveTimeout(TimeSpan.FromSeconds(30));
}
private void Query()
{
try
{
var max = _currentOffset - _lastEventMessageId;
if (max > 0)
{
var start = (MessageId)MessageIdUtils.GetMessageId(_lastEventMessageId);
var end = (MessageId)MessageIdUtils.GetMessageId(_currentOffset);
var query =
$"select {string.Join(", ", _message.Columns)}, __message_id__, __publish_time__, __properties__, __key__, __producer_name__, __sequence_id__, __partition__ from \"{_message.Topic}\" where __partition__ = {_partitionIndex} AND CAST(split_part(replace(replace(__message_id__, '('), ')'), ',', 1) AS BIGINT) BETWEEN bigint '{start.LedgerId}' AND bigint '{end.LedgerId}' AND CAST(split_part(replace(replace(__message_id__, '('), ')'), ',', 2) AS BIGINT) BETWEEN bigint '{start.EntryId + 1}' AND bigint '{end.EntryId}' ORDER BY __publish_time__ ASC LIMIT {_queryRange}";
var options = _message.Options;
options.Catalog = "pulsar";
options.Schema = "" + _message.Tenant + "/" + _message.Namespace + "";
options.Execute = query;
var session = options.ToClientSession();
var executor = new Executor(session, options, _self, _log);
_log.Info($"Executing: {options.Execute}");
_ = executor.Run().GetAwaiter().GetResult();
_lastEventMessageId = _currentOffset;
}
}
catch (Exception ex)
{
_log.Error(ex.ToString());
}
finally
{
_queryCancelable = _scheduler.ScheduleOnceCancelable(TimeSpan.FromSeconds(10), Query);
}
}
private void LiveConsume()
{
Receive <DataResponse>(c =>
{
for (var i = 0; i < c.Data.Count; i++)
{
var msgData = c.Data.ElementAt(i);
var msg = msgData["__message_id__"].ToString().Trim('(', ')').Split(',').Select(int.Parse).ToArray();
var messageId = MessageIdUtils.GetOffset(new MessageId(msg[0], msg[1], msg[2]));
var eventMessage = new EventEnvelope(msgData, messageId, _topicName.ToString());
_buffer.Post(eventMessage);
_currentOffset = messageId;
}
_buffer.Post(new EventStats(new StatsResponse(c.StatementStats)));
});
Receive <StatsResponse>(s =>
{
var stats = new EventStats(s);
_buffer.Post(stats);
});
Receive <ErrorResponse>(s =>
{
var error = new EventError(s);
_buffer.Post(error);
});
_queryCancelable = _scheduler.ScheduleOnceCancelable(TimeSpan.FromSeconds(60), Query);
}
private void Consume()
{
Receive<DataResponse>(c =>
{
for (var i = 0; i < c.Data.Count; i++)
{
var msgData = c.Data.ElementAt(i);
var msg = msgData["__message_id__"].ToString().Trim('(', ')').Split(',').Select(int.Parse).ToArray();
var messageId = MessageIdUtils.GetOffset(new MessageId(msg[0], msg[1], msg[2]));
if (messageId <= _toMessageId)
{
var eventMessage = new EventEnvelope(msgData, messageId, _topicName.ToString());
_buffer.Post(eventMessage);
_currentOffset = messageId;
}
else Self.GracefulStop(TimeSpan.FromSeconds(5));
}
_buffer.Post(new EventStats(new StatsResponse(c.StatementStats)));
});
Receive<StatsResponse>(s =>
{
var stats = new EventStats(s);
_buffer.Post(stats);
});
Receive<ErrorResponse>(s =>
{
var error = new EventError(s);
_buffer.Post(error);
});
Receive<ReceiveTimeout>(t => { Self.GracefulStop(TimeSpan.FromSeconds(5)); });
//to track last sequence id for lagging player
Context.SetReceiveTimeout(TimeSpan.FromSeconds(30));
}
public virtual async Task ReaderSourceTaggedTest()
{
var topic = $"reader-topics-{Guid.NewGuid()}";
var ids = PublishMessages(topic, 50);
var start = MessageIdUtils.GetOffset(ids.First());
var end = MessageIdUtils.GetOffset(ids.Last());
var conf = new ReaderConfigBuilder <DataOp>().Topic(topic);
var reader = _pulsarSystem.EventSource("public", "default", topic, start, end, "http://127.0.0.1:8080")
.Reader(_clientConfigurationData, conf, AvroSchema <DataOp> .Of(typeof(DataOp)))
.SourceMethod()
.CurrentTaggedEvents(new Messages.Consumer.Tag("twitter", "mestical"));
//let leave some time to wire everything up
await Task.Delay(TimeSpan.FromSeconds(20));
var receivedCount = 0;
while (receivedCount == 0)
{
await foreach (var response in reader.CurrentEvents(TimeSpan.FromSeconds(5)))
{
receivedCount++;
_output.WriteLine(JsonSerializer.Serialize(response, new JsonSerializerOptions {
WriteIndented = true
}));
}
await Task.Delay(TimeSpan.FromSeconds(1));
}
Assert.True(receivedCount > 0);
}
public void TestId(long ledger, long entry)
{
var id = new MessageId(ledger, entry, -1);
var offset = MessageIdUtils.GetOffset(id);
var id1 = MessageIdUtils.GetMessageId(offset);
Assert.Equal(id, id1);
}
private void LiveConsume()
{
Receive <ReceivedMessage <T> >(c =>
{
_parent.Tell(c);
_currentOffset = MessageIdUtils.GetOffset(c.Message.MessageId);
});
_flowSenderCancelable = _scheduler.ScheduleOnceCancelable(TimeSpan.FromSeconds(60), SendFlow);
}
//[Fact]
public virtual void SqlSourceTaggedTest()
{
var topic = $"presto-topics-{Guid.NewGuid()}";
var ids = PublishMessages(topic, 50);
var start = MessageIdUtils.GetOffset(ids.First());
var end = MessageIdUtils.GetOffset(ids.Last());
var cols = new HashSet <string> {
"text", "EventTime"
};
var option = new ClientOptions {
Server = "http://127.0.0.1:8081"
};
var source = _pulsarSystem.EventSource("public", "default", topic, start, end, "http://127.0.0.1:8080")
.Sql(option, cols)
.SourceMethod()
.CurrentTaggedEvents(new Messages.Consumer.Tag("twitter", "mestical"));
var receivedCount = 0;
for (var i = 0; i < 45; i++)
{
var response = source.CurrentEvents();
foreach (var data in response)
{
i++;
receivedCount++;
switch (data)
{
case EventEnvelope dr:
_output.WriteLine(JsonSerializer.Serialize(dr, new JsonSerializerOptions {
WriteIndented = true
}));
break;
case EventStats sr:
_output.WriteLine(JsonSerializer.Serialize(sr.Stats, new JsonSerializerOptions {
WriteIndented = true
}));
break;
case EventError er:
_output.WriteLine(JsonSerializer.Serialize(er.Error, new JsonSerializerOptions {
WriteIndented = true
}));
break;
}
}
if (receivedCount == 0)
{
Thread.Sleep(TimeSpan.FromSeconds(10));
}
}
Assert.True(receivedCount > 0);
}
public PrestoSourceActor(BufferBlock <IEventEnvelope> buffer, bool isLive, IPrestoEventSourceMessage message)
{
_buffer = buffer;
_self = Self;
_log = Context.GetLogger();
_scheduler = Context.System.Scheduler.Advanced;
_topicName = TopicName.Get(message.Topic);
_message = message;
_toMessageId = message.ToMessageId;
_lastEventMessageId = _message.FromMessageId;
_partitionIndex = ((MessageId)MessageIdUtils.GetMessageId(message.FromMessageId)).PartitionIndex;
_queryRange = message.ToMessageId - message.FromMessageId;
FirstQuery(isLive);
}
private void LiveConsume()
{
Receive <ReceivedMessage <T> >(c =>
{
var props = c.Message.Properties;
var tagged = props.FirstOrDefault(x => x.Key.Equals(_tag.Key, StringComparison.OrdinalIgnoreCase) && x.Value.Equals(_tag.Value, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrWhiteSpace(tagged.Value))
{
Context.Parent.Tell(c);
}
_currentOffset = MessageIdUtils.GetOffset(c.Message.MessageId);
//_sequenceId++;
});
_flowSenderCancelable = _scheduler.ScheduleOnceCancelable(TimeSpan.FromSeconds(60), SendFlow);
}
private void Consume()
{
Receive <ReceivedMessage <T> >(m =>
{
var c = m.Message;
var messageId = MessageIdUtils.GetOffset(m.Message.MessageId);
if (messageId <= _toOffset)
{
_parent.Tell(m);
_child.Tell(new AcknowledgeMessage <T>(c));
_child.Tell(new MessageProcessed <T>(c));
//_sequenceId++;
}
else
{
Self.GracefulStop(TimeSpan.FromSeconds(5));
}
});
Receive <ReceiveTimeout>(t => { Self.GracefulStop(TimeSpan.FromSeconds(5)); });
//to track last sequence id for lagging player
Context.SetReceiveTimeout(TimeSpan.FromSeconds(30));
}
private MessageId GetMessageId()
{
return((MessageId)MessageIdUtils.GetMessageId(_message.FromMessageId));
}
