public IKTable <Windowed <K>, VR> Aggregate <VR>(Initializer <VR> initializer, Aggregator <K, V, VR> aggregator, Materialized <K, VR, WindowStore <Bytes, byte[]> > materialized, string named = null)
{
CheckIfParamNull(initializer, "initializer");
CheckIfParamNull(aggregator, "aggregator");
materialized = materialized ?? Materialized <K, VR, WindowStore <Bytes, byte[]> > .Create();
if (materialized.KeySerdes == null)
{
materialized.WithKeySerdes(KeySerdes);
}
string name = new Named(named).OrElseGenerateWithPrefix(builder, KGroupedStream.AGGREGATE_NAME);
materialized.UseProvider(builder, KGroupedStream.AGGREGATE_NAME);
var aggSupplier = new KStreamWindowAggregate <K, V, VR, W>(
windowOptions,
materialized.StoreName,
initializer,
aggregator);
ISerDes <Windowed <K> > windowSerdes = materialized.KeySerdes != null ? new TimeWindowedSerDes <K>(materialized.KeySerdes, windowOptions.Size) : null;
return(aggBuilder.BuildWindow(name,
new TimestampedWindowStoreMaterializer <K, VR, W>(windowOptions, materialized).Materialize(),
aggSupplier,
materialized.QueryableStoreName,
windowSerdes,
materialized.ValueSerdes));
}
public IKStream <K, VR> MapValues <VR>(IValueMapperWithKey <K, V, VR> mapper, string named = null)
{
if (mapper == null)
{
throw new ArgumentNullException($"Mapper function can't be null");
}
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, MAPVALUES_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamMapValues <K, V, VR>(mapper), name);
ProcessorGraphNode <K, V> mapValuesProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
mapValuesProcessorNode.ValueChangingOperation = true;
builder.AddGraphNode(this.node, mapValuesProcessorNode);
// value serde cannot be preserved
return(new KStream <K, VR>(
name,
this.keySerdes,
null,
this.setSourceNodes,
mapValuesProcessorNode,
builder));
}
public KStream <K, VR> Join <K, V, V0, VR>(
KStream <K, V> joinLeft,
KStream <K, V0> joinRight,
IValueJoiner <V, V0, VR> joiner,
JoinWindowOptions windows,
Joined <K, V, V0> joined)
{
var named = new Named(joined.Name);
var joinThisSuffix = rightOuter ? "-outer-this-join" : "-this-join";
var joinOtherSuffix = leftOuter ? "-outer-other-join" : "-other-join";
var thisWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME);
var otherWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME);
var joinThisGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME);
var joinOtherGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME);
var joinThisName = named.SuffixWithOrElseGet(joinThisSuffix, joinThisGeneratedName);
var joinOtherName = named.SuffixWithOrElseGet(joinOtherSuffix, joinOtherGeneratedName);
var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME);
StreamGraphNode thisStreamsGraphNode = joinLeft.Node;
StreamGraphNode otherStreamsGraphNode = joinRight.Node;
var userProvidedBaseStoreName = joined.Name; // TODO : renamed, and create a StreamJoined DTO
// TODO TO FINISH
return(null);
}
internal IKTable <K, V> Table <K, V>(string topic, ConsumedInternal <K, V> consumed, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized)
{
var sourceName = new Named(consumed.Named).SuffixWithOrElseGet(TABLE_SOURCE_SUFFIX, this, KStream.SOURCE_NAME);
var tableSourceName = new Named(consumed.Named).OrElseGenerateWithPrefix(this, KTable.SOURCE_NAME);
KTableSource <K, V> tableSource = new KTableSource <K, V>(materialized.StoreName, materialized.QueryableStoreName);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(tableSource, tableSourceName);
var tableSourceNode = new TableSourceNode <K, V, IKeyValueStore <Bytes, byte[]> >(
topic, tableSourceName, sourceName, consumed,
materialized, processorParameters, false);
this.AddGraphNode(root, tableSourceNode);
return(new KTable <K, V, V>(tableSourceName,
consumed.KeySerdes,
consumed.ValueSerdes,
new List <string> {
sourceName
},
materialized.QueryableStoreName,
tableSource,
tableSourceNode,
this));
}
internal IGlobalKTable <K, V> GlobalTable <K, V>(string topic, ConsumedInternal <K, V> consumed, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentException("topic can't be null or empty", nameof(topic));
}
// explicitly disable logging for global stores
materialized.WithLoggingDisabled();
string sourceName = new Named(consumed.Named).SuffixWithOrElseGet(TABLE_SOURCE_SUFFIX, this, KStream.SOURCE_NAME);
string tableSourceName = new Named(consumed.Named).OrElseGenerateWithPrefix(this, KTable.SOURCE_NAME);
string storeName = materialized.StoreName;
// enforce store name as queryable name to always materialize global table stores
var tableSource = new KTableSource <K, V>(storeName, storeName);
var processorParameters = new ProcessorParameters <K, V>(tableSource, tableSourceName);
var tableSourceNode = new TableSourceNode <K, V, IKeyValueStore <Bytes, byte[]> >(
topic, tableSourceName, sourceName, consumed,
materialized, processorParameters, true);
this.AddGraphNode(root, tableSourceNode);
return(new GlobalKTable <K, V>(new KTableSourceValueGetterSupplier <K, V>(storeName), materialized.QueryableStoreName));
}
private IKStream <K, V>[] DoBranch(string named = null, params Func <K, V, bool>[] predicates)
{
var namedInternal = new Named(named);
if (predicates.Length == 0)
{
throw new ArgumentException("branch() requires at least one predicate");
}
String branchName = namedInternal.OrElseGenerateWithPrefix(this.builder, BRANCH_NAME);
String[] childNames = new String[predicates.Length];
for (int i = 0; i < predicates.Length; i++)
{
childNames[i] = namedInternal.SuffixWithOrElseGet($"predicate-{i}", this.builder, BRANCHCHILD_NAME);
}
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamBranch <K, V>(predicates, childNames), branchName);
ProcessorGraphNode <K, V> branchNode = new ProcessorGraphNode <K, V>(branchName, processorParameters);
this.builder.AddGraphNode(node, branchNode);
IKStream <K, V>[] branchChildren = new IKStream <K, V> [predicates.Length];
for (int i = 0; i < predicates.Length; i++)
{
ProcessorParameters <K, V> innerProcessorParameters = new ProcessorParameters <K, V>(new PassThrough <K, V>(), childNames[i]);
ProcessorGraphNode <K, V> branchChildNode = new ProcessorGraphNode <K, V>(childNames[i], innerProcessorParameters);
builder.AddGraphNode(branchNode, branchChildNode);
branchChildren[i] = new KStream <K, V>(childNames[i], this.keySerdes, this.valueSerdes, setSourceNodes, branchChildNode, builder);
}
return(branchChildren);
}
private void DoTo(ITopicNameExtractor <K, V> topicExtractor, Produced <K, V> produced)
{
string name = new Named(produced.Named).OrElseGenerateWithPrefix(this.builder, SINK_NAME);
StreamSinkNode <K, V> sinkNode = new StreamSinkNode <K, V>(topicExtractor, name, produced);
this.builder.AddGraphNode(node, sinkNode);
}
public void Print(Printed <K, V> printed)
{
var name = new Named(printed.Name).OrElseGenerateWithPrefix(this.builder, PRINTING_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(printed.Build(this.nameNode), name);
ProcessorGraphNode <K, V> printNode = new ProcessorGraphNode <K, V>(name, processorParameters);
builder.AddGraphNode(node, printNode);
}
public void Foreach(Action <K, V> action, string named = null)
{
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, FOREACH_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, false), name);
ProcessorGraphNode <K, V> foreachNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, foreachNode);
}
private IKStream <K, V> DoFilter(Func <K, V, bool> predicate, string named, bool not)
{
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, FILTER_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFilter <K, V>(predicate, not), name);
ProcessorGraphNode <K, V> filterProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, filterProcessorNode);
return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, filterProcessorNode, this.builder));
}
internal KStream <K, V> Stream <K, V>(string topic, ConsumedInternal <K, V> consumed)
{
var name = new Named(consumed.Named).OrElseGenerateWithPrefix(this, KStream.SOURCE_NAME);
var node = new StreamSourceNode <K, V>(topic, name, consumed);
this.AddGraphNode(root, node);
KStream <K, V> stream = new KStream <K, V>(name, consumed.KeySerdes, consumed.ValueSerdes, new List <string> {
name
}, node, this);
return(stream);
}
public void Foreach(Action <K, V> action, string named = null)
{
if (action == null)
{
throw new ArgumentNullException("Foreach() doesn't allow null action function ");
}
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, FOREACH_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, false), name);
ProcessorGraphNode <K, V> foreachNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, foreachNode);
}
private ProcessorGraphNode <K, V> InternalSelectKey <KR>(IKeyValueMapper <K, V, KR> mapper, string named = null)
{
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, KEY_SELECT_NAME);
WrappedKeyValueMapper <K, V, KeyValuePair <KR, V> > internalMapper =
new WrappedKeyValueMapper <K, V, KeyValuePair <KR, V> >(
(key, value) => new KeyValuePair <KR, V>(mapper.Apply(key, value), value));
KStreamMap <K, V, KR, V> kStreamMap = new KStreamMap <K, V, KR, V>(internalMapper);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(kStreamMap, name);
return(new ProcessorGraphNode <K, V>(name, processorParameters));
}
public void Print(Printed <K, V> printed)
{
if (printed == null)
{
throw new ArgumentNullException("Print() doesn't allow null printed instance");
}
var name = new Named(printed.Name).OrElseGenerateWithPrefix(this.builder, PRINTING_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(printed.Build(this.nameNode), name);
ProcessorGraphNode <K, V> printNode = new ProcessorGraphNode <K, V>(name, processorParameters);
builder.AddGraphNode(node, printNode);
}
public IKStream <KR, VR> FlatMap <KR, VR>(IKeyValueMapper <K, V, IEnumerable <KeyValuePair <KR, VR> > > mapper, string named = null)
{
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, FLATMAP_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFlatMap <K, V, KR, VR>(mapper), name);
ProcessorGraphNode <K, V> flatMapNode = new ProcessorGraphNode <K, V>(name, processorParameters);
flatMapNode.KeyChangingOperation = true;
builder.AddGraphNode(node, flatMapNode);
// key and value serde cannot be preserved
return(new KStream <KR, VR>(name, null, null, setSourceNodes, flatMapNode, builder));
}
private IKStream <K, V> DoFilter(Func <K, V, bool> predicate, string named, bool not)
{
if (predicate == null)
{
throw new ArgumentNullException($"Filter() doesn't allow null predicate function");
}
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, FILTER_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFilter <K, V>(predicate, not), name);
ProcessorGraphNode <K, V> filterProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, filterProcessorNode);
return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, filterProcessorNode, this.builder));
}
public IKStream <K, V> Peek(Action <K, V> action, string named = null)
{
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, PEEK_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, true), name);
ProcessorGraphNode <K, V> peekNode = new ProcessorGraphNode <K, V>(name, processorParameters);
builder.AddGraphNode(node, peekNode);
return(new KStream <K, V>(
name,
keySerdes,
valueSerdes,
setSourceNodes,
peekNode,
builder));
}
public IKStream <K, VR> FlatMapValues <VR>(IValueMapperWithKey <K, V, IEnumerable <VR> > mapper, string named = null)
{
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, FLATMAPVALUES_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFlatMapValues <K, V, VR>(mapper), name);
ProcessorGraphNode <K, V> flatMapValuesNode = new ProcessorGraphNode <K, V>(name, processorParameters);
flatMapValuesNode.ValueChangingOperation = true;
builder.AddGraphNode(this.node, flatMapValuesNode);
// value serde cannot be preserved
return(new KStream <K, VR>(
name,
this.keySerdes,
null,
this.setSourceNodes,
flatMapValuesNode,
builder));
}
public IKStream <KR, VR> Map <KR, VR>(IKeyValueMapper <K, V, KeyValuePair <KR, VR> > mapper, string named = null)
{
if (mapper == null)
{
throw new ArgumentNullException($"Map() doesn't allow null mapper function");
}
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, MAP_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamMap <K, V, KR, VR>(mapper), name);
ProcessorGraphNode <K, V> mapProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
mapProcessorNode.KeyChangingOperation = true;
builder.AddGraphNode(node, mapProcessorNode);
// key and value serde cannot be preserved
return(new KStream <KR, VR>(
name,
null,
null,
setSourceNodes,
mapProcessorNode,
builder));
}
public IKTable <Windowed <K>, V> Reduce(Reducer <V> reducer, Materialized <K, V, WindowStore <Bytes, byte[]> > materialized, string named = null)
{
CheckIfParamNull(reducer, "reducer");
materialized = materialized ?? Materialized <K, V, WindowStore <Bytes, byte[]> > .Create();
if (materialized.KeySerdes == null)
{
materialized.WithKeySerdes(KeySerdes);
}
if (materialized.ValueSerdes == null)
{
materialized.WithValueSerdes(ValueSerdes);
}
string name = new Named(named).OrElseGenerateWithPrefix(builder, KGroupedStream.REDUCE_NAME);
materialized.UseProvider(builder, KGroupedStream.REDUCE_NAME);
var aggSupplier = new KStreamWindowAggregate <K, V, V, W>(
windowOptions,
materialized.StoreName,
() => default,
public KStream <K, VR> Join <K, V, V0, VR>(
KStream <K, V> joinLeft,
KStream <K, V0> joinRight,
IValueJoiner <V, V0, VR> joiner,
JoinWindowOptions windows,
StreamJoinProps <K, V, V0> joined)
{
var named = new Named(joined.Name);
var joinLeftSuffix = rightOuter ? "-outer-this-join" : "-this-join";
var joinRightSuffix = leftOuter ? "-outer-other-join" : "-other-join";
var leftWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME);
var rightWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME);
var joinLeftGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME);
var joinRightGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME);
var joinLeftName = named.SuffixWithOrElseGet(joinLeftSuffix, joinLeftGeneratedName);
var joinRightName = named.SuffixWithOrElseGet(joinRightSuffix, joinRightGeneratedName);
var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME);
StreamGraphNode leftStreamsGraphNode = joinLeft.Node;
StreamGraphNode rightStreamsGraphNode = joinRight.Node;
var userProvidedBaseStoreName = joined.StoreName;
var leftStoreSupplier = joined.LeftStoreSupplier;
var rightStoreSupplier = joined.RightStoreSupplier;
StoreBuilder <WindowStore <K, V> > leftWindowStore;
StoreBuilder <WindowStore <K, V0> > rightWindowStore;
AssertUniqueStoreNames(leftStoreSupplier, rightStoreSupplier);
if (leftStoreSupplier == null)
{
var thisJoinStoreName = userProvidedBaseStoreName == null ? joinLeftGeneratedName : userProvidedBaseStoreName + joinLeftSuffix;
leftWindowStore = JoinWindowStoreBuilder(thisJoinStoreName, windows, joined.KeySerdes, joined.LeftValueSerdes);
}
else
{
AssertWindowSettings(leftStoreSupplier, windows);
leftWindowStore = Stores.WindowStoreBuilder(leftStoreSupplier, joined.KeySerdes, joined.LeftValueSerdes);
}
if (rightStoreSupplier == null)
{
var otherJoinStoreName = userProvidedBaseStoreName == null ? joinRightGeneratedName : userProvidedBaseStoreName + joinRightSuffix;
rightWindowStore = JoinWindowStoreBuilder(otherJoinStoreName, windows, joined.KeySerdes, joined.RightValueSerdes);
}
else
{
AssertWindowSettings(rightStoreSupplier, windows);
rightWindowStore = Stores.WindowStoreBuilder(rightStoreSupplier, joined.KeySerdes, joined.RightValueSerdes);
}
var leftStream = new KStreamJoinWindow <K, V>(leftWindowStore.Name);
var leftStreamParams = new ProcessorParameters <K, V>(leftStream, leftWindowStreamProcessorName);
var leftNode = new ProcessorGraphNode <K, V>(leftWindowStreamProcessorName, leftStreamParams);
builder.AddGraphNode(leftStreamsGraphNode, leftNode);
var rightStream = new KStreamJoinWindow <K, V0>(rightWindowStore.Name);
var rightStreamParams = new ProcessorParameters <K, V0>(rightStream, rightWindowStreamProcessorName);
var rightNode = new ProcessorGraphNode <K, V0>(rightWindowStreamProcessorName, rightStreamParams);
builder.AddGraphNode(rightStreamsGraphNode, rightNode);
var joinL = new KStreamKStreamJoin <K, V, V0, VR>(joinLeftName, rightWindowStore.Name, windows.beforeMs, windows.afterMs, joiner, leftOuter);
var joinLParams = new ProcessorParameters <K, V>(joinL, joinLeftName);
var joinR = new KStreamKStreamJoin <K, V0, V, VR>(joinRightName, leftWindowStore.Name, windows.beforeMs, windows.afterMs, joiner.Reverse(), rightOuter);
var joinRParams = new ProcessorParameters <K, V0>(joinR, joinRightName);
var merge = new PassThrough <K, VR>();
var mergeParams = new ProcessorParameters <K, VR>(merge, joinMergeName);
var joinNode = new StreamStreamJoinNode <K, V, V0, VR>(
joinMergeName,
joiner,
joinLParams,
joinRParams,
mergeParams,
leftStreamParams,
rightStreamParams,
leftWindowStore,
rightWindowStore,
joined);
builder.AddGraphNode(new List <StreamGraphNode> {
leftStreamsGraphNode, rightStreamsGraphNode
}, joinNode);
ISet <string> allSourceNodes = new HashSet <string>(joinLeft.SetSourceNodes);
allSourceNodes.AddRange(joinRight.SetSourceNodes);
return(new KStream <K, VR>(
joinMergeName,
joined.KeySerdes,
null,
allSourceNodes.ToList(),
joinNode,
builder));
}