public StreamTask(string threadId, TaskId id, TopicPartition partition, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer)
: base(id, partition, processorTopology, consumer, configuration)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.consumedOffsets = new Dictionary <TopicPartition, long>();
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
this.collector = new RecordCollector(logPrefix);
collector.Init(ref this.producer);
var sourceTimestampExtractor = (processorTopology.GetSourceProcessor(id.Topic) as ISourceProcessor).Extractor;
Context = new ProcessorContext(configuration, stateMgr).UseRecordCollector(collector);
processor = processorTopology.GetSourceProcessor(partition.Topic);
queue = new RecordQueue <ConsumeResult <byte[], byte[]> >(
100,
logPrefix,
$"record-queue-{id.Topic}-{id.Partition}",
sourceTimestampExtractor == null ? configuration.DefaultTimestampExtractor : sourceTimestampExtractor);
}
private Dictionary <TopicPartition, RecordQueue> GetPartitions()
{
var timestampEx = new FailOnInvalidTimestamp();
var partitions = new Dictionary <TopicPartition, RecordQueue>();
var sourceProcessor = new SourceProcessor <string, string>("source1", "test", serdes, serdes, timestampEx);
var sourceProcessor2 = new SourceProcessor <string, string>("source2", "test2", serdes, serdes, timestampEx);
var recordQueue = new RecordQueue("", "source_queue", timestampEx, topicPart1, sourceProcessor);
var recordQueue2 = new RecordQueue("", "source2_queue", timestampEx, topicPart2, sourceProcessor2);
partitions.Add(topicPart1, recordQueue);
partitions.Add(topicPart2, recordQueue2);
return(partitions);
}
public void CheckAddWhenMaximumSizeAttempt()
{
RecordQueue <string> queue = new RecordQueue <string>(10, "", "test", null);
for (int i = 0; i < 10; ++i)
{
queue.AddRecord($"test-{i}");
}
Assert.AreEqual(10, queue.Size);
bool b = queue.AddRecord($"test-too");
Assert.AreEqual(10, queue.Size);
Assert.IsFalse(b);
}
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions, ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration, IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer)
: base(id, partitions, processorTopology, consumer, configuration)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
consumedOffsets = new Dictionary <TopicPartition, long>();
maxTaskIdleMs = configuration.MaxTaskIdleMs;
maxBufferedSize = configuration.BufferedRecordsPerPartition;
followMetadata = configuration.FollowMetadata;
idleStartTime = -1;
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
collector = new RecordCollector(logPrefix, configuration, id);
collector.Init(ref this.producer);
Context = new ProcessorContext(this, configuration, stateMgr).UseRecordCollector(collector);
Context.FollowMetadata = followMetadata;
var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>();
foreach (var p in partitions)
{
var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic);
var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor;
var queue = new RecordQueue(
logPrefix,
$"record-queue-{p.Topic}-{id.Id}-{id.Partition}",
sourceTimestampExtractor,
p,
sourceProcessor);
partitionsQueue.Add(p, queue);
processors.Add(sourceProcessor);
}
partitionGrouper = new PartitionGrouper(partitionsQueue);
}
public void QueueOneMessageTest()
{
var timestampEx = new FailOnInvalidTimestamp();
var serdes = new StringSerDes();
var sourceProcessor = new SourceProcessor <string, string>("source", "test", serdes, serdes, timestampEx);
var recordQueue = new RecordQueue("", "", timestampEx, new TopicPartition("test", 0), sourceProcessor);
int size = recordQueue.Queue(new ConsumeResult <byte[], byte[]>()
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize("key", new SerializationContext()),
Value = serdes.Serialize("test", new SerializationContext())
}
});
Assert.AreEqual(1, size);
}
private void ProcessTweet(ITweet tw)
{
Console.WriteLine("A tweet has been found");
if (tw.RetweetedTweet != null)
{
tw = tw.RetweetedTweet;
Console.WriteLine("Extracting main Tweet rather than RT");
}
Console.WriteLine("The tweet is " + tw.FullText);
List <IMediaEntity> Medias = tw.Media;
if (Medias != null && Medias.Count > 0)
{
Application.Current.Dispatcher.Invoke(delegate
{
IMediaEntity m = Medias[random.Next(Medias.Count)];
if (m.MediaURL.Contains(".jpg"))
{
TextURL = tw.Url;
CurrentImageUrl = m.MediaURL;
if (IsShowImage)
{
ImageFromTweet = new BitmapImage(new Uri(CurrentImageUrl));
}
}
});
if (!Mongo_isTweetSaved(tw.IdStr))
{
Mongo_SaveNewTweet(tw);
RecordQueue.Enqueue(tw);
//GCS_SaveNewTweetImage(tw);
}
}
}
public void QueueDequeueOneMessageTest()
{
var timestampEx = new FailOnInvalidTimestamp();
var serdes = new StringSerDes();
var sourceProcessor = new SourceProcessor <string, string>("source", "test", serdes, serdes, timestampEx);
var recordQueue = new RecordQueue("", "", timestampEx, new TopicPartition("test", 0), sourceProcessor);
recordQueue.Queue(new ConsumeResult <byte[], byte[]>()
{
Message = new Message <byte[], byte[]>
{
Key = serdes.Serialize("key", new SerializationContext()),
Value = serdes.Serialize("test", new SerializationContext())
}
});
var r = recordQueue.Poll();
Assert.IsNotNull(r);
Assert.AreEqual("key", serdes.Deserialize(r.Message.Key, new SerializationContext()));
Assert.AreEqual("test", serdes.Deserialize(r.Message.Value, new SerializationContext()));
Assert.AreEqual(0, recordQueue.Size);
Assert.IsTrue(recordQueue.IsEmpty);
}
public void CheckMaximumSize()
{
RecordQueue <string> queue = new RecordQueue <string>(10, "", "test", null);
Assert.AreEqual(10, queue.MaxSize);
}
public StreamTask(string threadId, TaskId id, IEnumerable <TopicPartition> partitions,
ProcessorTopology processorTopology, IConsumer <byte[], byte[]> consumer, IStreamConfig configuration,
IKafkaSupplier kafkaSupplier, IProducer <byte[], byte[]> producer, IChangelogRegister changelogRegister,
StreamMetricsRegistry streamMetricsRegistry)
: base(id, partitions, processorTopology, consumer, configuration, changelogRegister)
{
this.threadId = threadId;
this.kafkaSupplier = kafkaSupplier;
this.streamMetricsRegistry = streamMetricsRegistry;
consumedOffsets = new Dictionary <TopicPartition, long>();
maxTaskIdleMs = configuration.MaxTaskIdleMs;
maxBufferedSize = configuration.BufferedRecordsPerPartition;
followMetadata = configuration.FollowMetadata;
idleStartTime = -1;
// eos enabled
if (producer == null)
{
this.producer = CreateEOSProducer();
InitializeTransaction();
eosEnabled = true;
}
else
{
this.producer = producer;
}
var droppedRecordsSensor = TaskMetrics.DroppedRecordsSensor(this.threadId, Id, this.streamMetricsRegistry);
collector = new RecordCollector(logPrefix, configuration, id, droppedRecordsSensor);
collector.Init(ref this.producer);
Context = new ProcessorContext(this, configuration, stateMgr, streamMetricsRegistry)
.UseRecordCollector(collector);
Context.FollowMetadata = followMetadata;
var partitionsQueue = new Dictionary <TopicPartition, RecordQueue>();
foreach (var p in partitions)
{
var sourceProcessor = processorTopology.GetSourceProcessor(p.Topic);
sourceProcessor.SetTaskId(id);
var sourceTimestampExtractor = sourceProcessor.Extractor ?? configuration.DefaultTimestampExtractor;
var queue = new RecordQueue(
logPrefix,
$"record-queue-{p.Topic}-{id.Id}-{id.Partition}",
sourceTimestampExtractor,
p,
sourceProcessor,
droppedRecordsSensor);
partitionsQueue.Add(p, queue);
processors.Add(sourceProcessor);
}
partitionGrouper = new PartitionGrouper(partitionsQueue);
closeTaskSensor = ThreadMetrics.ClosedTaskSensor(this.threadId, streamMetricsRegistry);
activeBufferedRecordSensor = TaskMetrics.ActiveBufferedRecordsSensor(this.threadId, Id, streamMetricsRegistry);
processSensor = TaskMetrics.ProcessSensor(this.threadId, Id, streamMetricsRegistry);
processLatencySensor = TaskMetrics.ProcessLatencySensor(this.threadId, Id, streamMetricsRegistry);
enforcedProcessingSensor = TaskMetrics.EnforcedProcessingSensor(this.threadId, Id, streamMetricsRegistry);
commitSensor = TaskMetrics.CommitSensor(this.threadId, Id, streamMetricsRegistry);
activeRestorationSensor = TaskMetrics.ActiveRestorationSensor(this.threadId, Id, streamMetricsRegistry);
restorationRecordsSendsor = TaskMetrics.RestorationRecordsSensor(this.threadId, Id, streamMetricsRegistry);
}
