internal void AddGlobalStore <K, V, S>(string topicName,
StoreBuilder <S> storeBuilder,
string sourceName,
ConsumedInternal <K, V> consumed,
ProcessorParameters <K, V> processorParameters) where S : IStateStore
{
string processorName = processorParameters.ProcessorName;
ValidateGlobalStoreArguments(sourceName, topicName, processorName, processorParameters.Processor, storeBuilder.Name, storeBuilder.LoggingEnabled);
ValidateTopicNotAlreadyRegistered(topicName);
var predecessors = new[] { sourceName };
var nodeFactory = new ProcessorNodeFactory <K, V>(processorName, predecessors, processorParameters.Processor);
globalTopics.Add(topicName);
nodeFactories.Add(sourceName, new SourceNodeFactory <K, V>(sourceName, topicName, consumed.TimestampExtractor, consumed.KeySerdes, consumed.ValueSerdes));
// TODO: ?
// nodeToSourceTopics.put(sourceName, Arrays.asList(topics));
nodeGrouper.Add(sourceName);
nodeFactory.AddStateStore(storeBuilder.Name);
nodeFactories.Add(processorName, nodeFactory);
nodeGrouper.Add(processorName);
nodeGrouper.Unite(processorName, predecessors);
globalStateBuilders.Add(storeBuilder.Name, storeBuilder);
ConnectSourceStoreAndTopic(storeBuilder.Name, topicName);
nodeGroups = null;
}
public TableSourceNode(string topicName, string streamGraphNode,
string sourceName, ConsumedInternal <K, V> consumed,
Materialized <K, V, S> materialized, ProcessorParameters <K, V> processorParameters, bool isGlobalKTable = false)
: base(topicName, streamGraphNode, consumed)
{
this.materialized = materialized;
this.processorParameters = processorParameters;
this.sourceName = sourceName;
this.isGlobalKTable = isGlobalKTable;
}
/// <summary>
/// Create a <see cref="IGlobalKTable{K,V}"/> for the specified topic.
/// Input keyvalue records with <code>null</code> key will be dropped.
/// The resulting <see cref="IGlobalKTable{K,V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// However, no internal changelog topic is created since the original input topic can be used for recovery.
/// Note that <see cref="IGlobalKTable{K,V}"/> always applies <code>"auto.offset.reset"</code> strategy <code>"earliest"</code>
/// regardless of the specified value in <see cref="IStreamConfig"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IGlobalKTable{K,V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IGlobalKTable <K, V> GlobalTable <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor)
{
materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create();
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
materialized.UseProvider(internalStreamBuilder, $"{topic}-")?.InitConsumed(consumedInternal);
return(internalStreamBuilder.GlobalTable(topic, consumedInternal, materialized));
}
/// <summary>
/// Create a <see cref="KStream{K, V}"/> from the specified topic.
/// The default "auto.offset.reset" strategy and default <see cref="ITimestampExtractor"/> as specified in the <see cref="IStreamConfig"/> are used.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case it is the user's responsibility to repartition the data before any key based operation
/// (like aggregation or join) is applied to the returned <see cref="IKStream{K, V}"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to used. If null the default timestamp extractor from config will be used</param>
/// <returns>A <see cref="IKStream{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKStream <K, V> Stream <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, string named, ITimestampExtractor extractor)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentException("Topic of KStream must not be null or empty");
}
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
return(internalStreamBuilder.Stream(topic, consumedInternal));
}
internal void AddSourceOperator <K, V>(string topic, string nameNode, ConsumedInternal <K, V> consumed)
{
if (!sourceOperators.ContainsKey(nameNode))
{
SourceProcessor <K, V> source = new SourceProcessor <K, V>(nameNode, topic, consumed.KeySerdes, consumed.ValueSerdes, consumed.TimestampExtractor);
sourceOperators.Add(nameNode, source);
}
else
{
throw new Exception("Source operator already exist !");
}
}
internal Materialized <K, V, S> InitConsumed(ConsumedInternal <K, V> consumed)
{
if (KeySerdes == null)
{
KeySerdes = consumed.KeySerdes;
}
if (ValueSerdes == null)
{
ValueSerdes = consumed.ValueSerdes;
}
return(this);
}
/// <summary>
/// Create a <see cref="IKTable{K, V}"/> for the specified topic.
/// Input keyvalue records with null key will be dropped.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case the returned <see cref="IKTable{K, V}"/> will be corrupted.
///
/// The resulting <see cref="IKTable{K, V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IKTable{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKTable <K, V> Table <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentException("Topic of KTable must not be null or empty");
}
materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create();
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
materialized.UseProvider(internalStreamBuilder, $"{topic}-").InitConsumed(consumedInternal);
return(internalStreamBuilder.Table(topic, consumedInternal, materialized));
}
internal void AddSourceOperator<K, V>(string topic, string nameNode, ConsumedInternal<K, V> consumed)
{
if (string.IsNullOrEmpty(topic))
throw new TopologyException("You must provide at least one topic");
if (nodeFactories.ContainsKey(nameNode))
throw new TopologyException($"Source processor {nameNode} is already added.");
if(sourcesTopics.Contains(topic)) {
throw new TopologyException($"Topic {topic} has already been registered by another source.");
}
sourcesTopics.Add(topic);
nodeFactories.Add(nameNode,
new SourceNodeFactory<K, V>(nameNode, topic, consumed.TimestampExtractor, consumed.KeySerdes, consumed.ValueSerdes));
nodeGrouper.Add(nameNode);
nodeGroups = null;
}
public StreamSourceNode(string topicName, string streamGraphNode, ConsumedInternal <K, V> consumed)
: base(streamGraphNode)
{
this.topicName = topicName;
this.consumed = consumed;
}
/// <summary>
/// Create a <see cref="KStream{K, V}"/> from the specified topic.
/// The default "auto.offset.reset" strategy and default <see cref="ITimestampExtractor"/> as specified in the <see cref="IStreamConfig"/> are used.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case it is the user's responsibility to repartition the data before any key based operation
/// (like aggregation or join) is applied to the returned <see cref="IKStream{K, V}"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to used. If null the default timestamp extractor from config will be used</param>
/// <returns>A <see cref="IKStream{K, V}"/> for the specified topic</returns>
public IKStream <K, V> Stream <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, string named, ITimestampExtractor extractor)
{
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
return(internalStreamBuilder.Stream(topic, consumedInternal));
}
