public RecordQueue(int maxSize, string logPrefix, string nameQueue, ITimestampExtractor timestampExtractor)
{
this.maxSize = maxSize;
this.logPrefix = logPrefix;
this.nameQueue = nameQueue;
this.timestampExtractor = timestampExtractor;
this.queue = new Queue <T>(this.maxSize);
}
public SourceNodeFactory(string name, string topic, ITimestampExtractor timestampExtractor, ISerDes <K> keySerdes, ISerDes <V> valueSerdes)
: base(name, new string[0])
{
Topic = topic;
Extractor = timestampExtractor;
KeySerdes = keySerdes;
ValueSerdes = valueSerdes;
}
public ConsumedInternal(
string named,
ISerDes <K> keySerdes,
ISerDes <V> valueSerdes,
ITimestampExtractor timestampExtractor)
{
Named = named;
KeySerdes = keySerdes;
ValueSerdes = valueSerdes;
TimestampExtractor = timestampExtractor;
}
public RecordQueue(
string logPrefix,
string nameQueue,
ITimestampExtractor timestampExtractor,
ISourceProcessor sourceProcessor)
{
this.logPrefix = $"{logPrefix}- recordQueue [{nameQueue}] ";
queue = new List <ConsumeResult <byte[], byte[]> >();
this.timestampExtractor = timestampExtractor;
this.sourceProcessor = sourceProcessor;
}
public RecordQueue(string logPrefix,
string nameQueue,
ITimestampExtractor timestampExtractor,
TopicPartition topicPartition,
ISourceProcessor sourceProcessor,
Sensor droppedRecordsSensor)
{
this.logPrefix = $"{logPrefix}- recordQueue [{nameQueue}] ";
queue = new List <ConsumeResult <byte[], byte[]> >();
this.timestampExtractor = timestampExtractor;
this.topicPartition = topicPartition;
this.sourceProcessor = sourceProcessor;
this.droppedRecordsSensor = droppedRecordsSensor;
}
private Topology KStreamWithImplicitReKeyJoinTopology(ITimestampExtractor timestampExtractor)
{
StringSerDes stringSerdes = new StringSerDes();
var builder = new StreamBuilder();
var userRegionsTable = builder.Table <string, string>(userRegionsTopic, stringSerdes, stringSerdes, InMemory <string, string> .As("table-store"), "table", timestampExtractor);
var userClicksStream = builder.Stream <string, string>(userClicksTopic, stringSerdes, stringSerdes, timestampExtractor);
userClicksStream
.SelectKey((k, v) => k)
.Join(userRegionsTable, Join)
.To(outputTopic);
return(builder.Build());
}
private Topology SimpleJoinTopology(ITimestampExtractor timestampExtractor)
{
StringSerDes stringSerdes = new StringSerDes();
var builder = new StreamBuilder();
// a simple use case with a KStream Topic that needs to join with a KTable
// internally KTable are versioned per timestamp.
// the timestamp used by default is ingestion time,
// but you can change that to your need to use event time (a field from the message) by configuring
// a timestamp extractor on the stream and / or table
// when doing a KStreams/KTable join the framework will look up for the value of a given key
// in the KTable at a timestamp <= to the timestamp of the event on the stream side
var userRegionsTable = builder.Table <string, string>(userRegionsTopic, stringSerdes, stringSerdes, InMemory <string, string> .As("table-store"), "table", timestampExtractor);
var userClicksStream = builder.Stream <string, string>(userClicksTopic, stringSerdes, stringSerdes, timestampExtractor);
userClicksStream
.Join(userRegionsTable, Join)
.To(outputTopic);
return(builder.Build());
}
/// <summary>
/// Create a <see cref="KStream{K, V}"/> from the specified topic.
/// The default "auto.offset.reset" strategy as specified in the <see cref="IStreamConfig"/> are used.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case it is the user's responsibility to repartition the data before any key based operation
/// (like aggregation or join) is applied to the returned <see cref="IKStream{K, V}"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="extractor">the timestamp extractor to used. If null the default timestamp extractor from config will be used</param>
/// <returns>A <see cref="IKStream{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKStream <K, V> Stream <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, ITimestampExtractor extractor)
=> this.Stream(topic, keySerdes, valueSerdes, null, extractor);
/// <summary>
/// Create a <see cref="IGlobalKTable{K,V}"/> for the specified topic.
/// Input keyvalue records with <code>null</code> key will be dropped.
/// The resulting <see cref="IGlobalKTable{K,V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// However, no internal changelog topic is created since the original input topic can be used for recovery.
/// Note that <see cref="IGlobalKTable{K,V}"/> always applies <code>"auto.offset.reset"</code> strategy <code>"earliest"</code>
/// regardless of the specified value in <see cref="IStreamConfig"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <typeparam name="KS">Key deserializer type</typeparam>
/// <typeparam name="VS">Value deserializer type</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IGlobalKTable{K,V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IGlobalKTable <K, V> GlobalTable <K, V, KS, VS>(string topic, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor)
where KS : ISerDes <K>, new()
where VS : ISerDes <V>, new()
=> GlobalTable(topic, new KS(), new VS(), materialized, named, extractor);
/// <summary>
/// Create a <see cref="IGlobalKTable{K,V}"/> for the specified topic.
/// Input keyvalue records with <code>null</code> key will be dropped.
/// The resulting <see cref="IGlobalKTable{K,V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// However, no internal changelog topic is created since the original input topic can be used for recovery.
/// Note that <see cref="IGlobalKTable{K,V}"/> always applies <code>"auto.offset.reset"</code> strategy <code>"earliest"</code>
/// regardless of the specified value in <see cref="IStreamConfig"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IGlobalKTable{K,V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IGlobalKTable <K, V> GlobalTable <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor)
{
materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create();
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
materialized.UseProvider(internalStreamBuilder, $"{topic}-")?.InitConsumed(consumedInternal);
return(internalStreamBuilder.GlobalTable(topic, consumedInternal, materialized));
}
/// <summary>
/// Create a <see cref="IKTable{K, V}"/> for the specified topic.
/// The default "auto.offset.reset" strategy, <see cref="ITimestampExtractor"/> as specified in the <see cref="IStreamConfig"/> are used.
/// Input keyvalue records with null key will be dropped.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case the returned <see cref="IKTable{K, V}"/> will be corrupted.
///
/// The resulting <see cref="IKTable{K, V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <typeparam name="KS">Key deserializer type</typeparam>
/// <typeparam name="VS">Value deserializer type</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="extractor">the timestamp extractor to used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IKTable{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKTable <K, V> Table <K, V, KS, VS>(string topic, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, ITimestampExtractor extractor)
where KS : ISerDes <K>, new()
where VS : ISerDes <V>, new()
=> Table <K, V, KS, VS>(topic, materialized, null, extractor);
/// <summary>
/// Create a <see cref="IKTable{K, V}"/> for the specified topic.
/// Input keyvalue records with null key will be dropped.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case the returned <see cref="IKTable{K, V}"/> will be corrupted.
///
/// The resulting <see cref="IKTable{K, V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IKTable{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKTable <K, V> Table <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentException("Topic of KTable must not be null or empty");
}
materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create();
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
materialized.UseProvider(internalStreamBuilder, $"{topic}-").InitConsumed(consumedInternal);
return(internalStreamBuilder.Table(topic, consumedInternal, materialized));
}
/// <summary>
/// Create a <see cref="KStream{K, V}"/> from the specified topic.
/// The default "auto.offset.reset" strategy, default <see cref="ITimestampExtractor"/> as specified in the <see cref="IStreamConfig"/> are used.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case it is the user's responsibility to repartition the data before any key based operation
/// (like aggregation or join) is applied to the returned <see cref="IKStream{K, V}"/>.
/// </summary>
/// <typeparam name="KS">Key deserializer type</typeparam>
/// <typeparam name="VS">Value deserializer type</typeparam>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to used. If null the default timestamp extractor from config will be used</param>
/// <returns>A <see cref="IKStream{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKStream <K, V> Stream <K, V, KS, VS>(string topic, string named, ITimestampExtractor extractor)
where KS : ISerDes <K>, new()
where VS : ISerDes <V>, new()
=> Stream(topic, new KS(), new VS(), named, extractor);
/// <summary>
/// Create a <see cref="KStream{K, V}"/> from the specified topic.
/// The default "auto.offset.reset" strategy, default <see cref="ITimestampExtractor"/> as specified in the <see cref="IStreamConfig"/> are used.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case it is the user's responsibility to repartition the data before any key based operation
/// (like aggregation or join) is applied to the returned <see cref="IKStream{K, V}"/>.
/// </summary>
/// <typeparam name="KS">Key deserializer type</typeparam>
/// <typeparam name="VS">Value deserializer type</typeparam>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="extractor">the timestamp extractor to used. If null the default timestamp extractor from config will be used</param>
/// <returns>A <see cref="IKStream{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKStream <K, V> Stream <K, V, KS, VS>(string topic, ITimestampExtractor extractor)
where KS : ISerDes <K>, new()
where VS : ISerDes <V>, new()
=> Stream <K, V, KS, VS>(topic, null, extractor);
private static StreamOptions Create(ITimestampExtractor extractor) => Create(extractor, null);
private static StreamOptions Create(ITimestampExtractor extractor, string named)
=> new StreamOptions()
{
Named = named, Extractor = extractor
};
private StreamOptions WithTimestrampExtractor(ITimestampExtractor extractor)
{
Extractor = extractor;
return(this);
}
internal SourceProcessor(string name, string topicName, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, ITimestampExtractor extractor)
: base(name, keySerdes, valueSerdes)
{
this.topicName = topicName;
Extractor = extractor;
}
/// <summary>
/// Create a <see cref="KStream{K, V}"/> from the specified topic.
/// The default "auto.offset.reset" strategy and default <see cref="ITimestampExtractor"/> as specified in the <see cref="IStreamConfig"/> are used.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case it is the user's responsibility to repartition the data before any key based operation
/// (like aggregation or join) is applied to the returned <see cref="IKStream{K, V}"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to used. If null the default timestamp extractor from config will be used</param>
/// <returns>A <see cref="IKStream{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKStream <K, V> Stream <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, string named, ITimestampExtractor extractor)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentException("Topic of KStream must not be null or empty");
}
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
return(internalStreamBuilder.Stream(topic, consumedInternal));
}
/// <summary>
/// Create a <see cref="KStream{K, V}"/> from the specified topic.
/// The default "auto.offset.reset" strategy and default <see cref="ITimestampExtractor"/> as specified in the <see cref="IStreamConfig"/> are used.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case it is the user's responsibility to repartition the data before any key based operation
/// (like aggregation or join) is applied to the returned <see cref="IKStream{K, V}"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to used. If null the default timestamp extractor from config will be used</param>
/// <returns>A <see cref="IKStream{K, V}"/> for the specified topic</returns>
public IKStream <K, V> Stream <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, string named, ITimestampExtractor extractor)
{
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
return(internalStreamBuilder.Stream(topic, consumedInternal));
}
