private IKStream <K, V>[] DoBranch(string named = null, params Func <K, V, bool>[] predicates)
{
var namedInternal = new Named(named);
if (predicates.Length == 0)
{
throw new ArgumentException("branch() requires at least one predicate");
}
String branchName = namedInternal.OrElseGenerateWithPrefix(this.builder, BRANCH_NAME);
String[] childNames = new String[predicates.Length];
for (int i = 0; i < predicates.Length; i++)
{
childNames[i] = namedInternal.SuffixWithOrElseGet($"predicate-{i}", this.builder, BRANCHCHILD_NAME);
}
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamBranch <K, V>(predicates, childNames), branchName);
ProcessorGraphNode <K, V> branchNode = new ProcessorGraphNode <K, V>(branchName, processorParameters);
this.builder.AddGraphNode(node, branchNode);
IKStream <K, V>[] branchChildren = new IKStream <K, V> [predicates.Length];
for (int i = 0; i < predicates.Length; i++)
{
ProcessorParameters <K, V> innerProcessorParameters = new ProcessorParameters <K, V>(new PassThrough <K, V>(), childNames[i]);
ProcessorGraphNode <K, V> branchChildNode = new ProcessorGraphNode <K, V>(childNames[i], innerProcessorParameters);
builder.AddGraphNode(branchNode, branchChildNode);
branchChildren[i] = new KStream <K, V>(childNames[i], this.keySerdes, this.valueSerdes, setSourceNodes, branchChildNode, builder);
}
return(branchChildren);
}
public IKStream <K, VR> MapValues <VR>(IValueMapperWithKey <K, V, VR> mapper, string named = null)
{
if (mapper == null)
{
throw new ArgumentNullException($"Mapper function can't be null");
}
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, MAPVALUES_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamMapValues <K, V, VR>(mapper), name);
ProcessorGraphNode <K, V> mapValuesProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
mapValuesProcessorNode.ValueChangingOperation = true;
builder.AddGraphNode(this.node, mapValuesProcessorNode);
// value serde cannot be preserved
return(new KStream <K, VR>(
name,
this.keySerdes,
null,
this.setSourceNodes,
mapValuesProcessorNode,
builder));
}
public void Print(Printed <K, V> printed)
{
var name = new Named(printed.Name).OrElseGenerateWithPrefix(this.builder, PRINTING_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(printed.Build(this.nameNode), name);
ProcessorGraphNode <K, V> printNode = new ProcessorGraphNode <K, V>(name, processorParameters);
builder.AddGraphNode(node, printNode);
}
public void Foreach(Action <K, V> action, string named = null)
{
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, FOREACH_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, false), name);
ProcessorGraphNode <K, V> foreachNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, foreachNode);
}
private IKStream <K, V> DoFilter(Func <K, V, bool> predicate, string named, bool not)
{
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, FILTER_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFilter <K, V>(predicate, not), name);
ProcessorGraphNode <K, V> filterProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, filterProcessorNode);
return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, filterProcessorNode, this.builder));
}
public void Foreach(Action <K, V> action, string named = null)
{
if (action == null)
{
throw new ArgumentNullException("Foreach() doesn't allow null action function ");
}
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, FOREACH_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, false), name);
ProcessorGraphNode <K, V> foreachNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, foreachNode);
}
public void Print(Printed <K, V> printed)
{
if (printed == null)
{
throw new ArgumentNullException("Print() doesn't allow null printed instance");
}
var name = new Named(printed.Name).OrElseGenerateWithPrefix(this.builder, PRINTING_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(printed.Build(this.nameNode), name);
ProcessorGraphNode <K, V> printNode = new ProcessorGraphNode <K, V>(name, processorParameters);
builder.AddGraphNode(node, printNode);
}
public IKStream <KR, VR> FlatMap <KR, VR>(IKeyValueMapper <K, V, IEnumerable <KeyValuePair <KR, VR> > > mapper, string named = null)
{
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, FLATMAP_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFlatMap <K, V, KR, VR>(mapper), name);
ProcessorGraphNode <K, V> flatMapNode = new ProcessorGraphNode <K, V>(name, processorParameters);
flatMapNode.KeyChangingOperation = true;
builder.AddGraphNode(node, flatMapNode);
// key and value serde cannot be preserved
return(new KStream <KR, VR>(name, null, null, setSourceNodes, flatMapNode, builder));
}
private IKStream <K, V> DoFilter(Func <K, V, bool> predicate, string named, bool not)
{
if (predicate == null)
{
throw new ArgumentNullException($"Filter() doesn't allow null predicate function");
}
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, FILTER_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFilter <K, V>(predicate, not), name);
ProcessorGraphNode <K, V> filterProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, filterProcessorNode);
return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, filterProcessorNode, this.builder));
}
public IKStream <K, V> ToStream(string named = null)
{
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, TOSTREAM_NAME);
var p = new WrapperValueMapperWithKey <K, Change <V>, V>((k, v) => v.NewValue);
IProcessorSupplier <K, Change <V> > processorMapValues = new KStreamMapValues <K, Change <V>, V>(p);
ProcessorParameters <K, Change <V> > processorParameters = new ProcessorParameters <K, Change <V> >(processorMapValues, name);
ProcessorGraphNode <K, Change <V> > toStreamNode = new ProcessorGraphNode <K, Change <V> >(name, processorParameters);
builder.AddGraphNode(this.node, toStreamNode);
// we can inherit parent key and value serde
return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, toStreamNode, builder));
}
private IKGroupedStream <KR, V> DoGroup <KR>(IKeyValueMapper <K, V, KR> keySelector, Grouped <KR, V> grouped)
{
ProcessorGraphNode <K, V> selectKeyMapNode = InternalSelectKey(keySelector, grouped.Named);
selectKeyMapNode.KeyChangingOperation = true;
builder.AddGraphNode(this.node, selectKeyMapNode);
return(new KGroupedStream <KR, V>(
selectKeyMapNode.streamGraphNode,
grouped,
this.setSourceNodes,
selectKeyMapNode,
builder));
}
public IKStream <K, V> Peek(Action <K, V> action, string named = null)
{
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, PEEK_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, true), name);
ProcessorGraphNode <K, V> peekNode = new ProcessorGraphNode <K, V>(name, processorParameters);
builder.AddGraphNode(node, peekNode);
return(new KStream <K, V>(
name,
keySerdes,
valueSerdes,
setSourceNodes,
peekNode,
builder));
}
public IKStream <KR, V> SelectKey <KR>(IKeyValueMapper <K, V, KR> mapper, string named = null)
{
ProcessorGraphNode <K, V> selectKeyProcessorNode = InternalSelectKey(mapper, named);
selectKeyProcessorNode.KeyChangingOperation = true;
builder.AddGraphNode(node, selectKeyProcessorNode);
// key serde cannot be preserved
return(new KStream <KR, V>(
selectKeyProcessorNode.streamGraphNode,
null,
valueSerdes,
setSourceNodes,
selectKeyProcessorNode,
builder));
}
public void WriteTopologyTest()
{
var builder = new InternalTopologyBuilder();
List <StreamGraphNode> nodes = new List <StreamGraphNode>();
RootNode root = new RootNode();
var source = new StreamSourceNode <string, string>(
"topic",
"source-01",
new Stream.Internal.ConsumedInternal <string, string>("source-01", new StringSerDes(),
new StringSerDes(), null));
root.AppendChild(source);
nodes.Add(source);
var filterParameters =
new ProcessorParameters <string, string>(
new KStreamFilter <string, string>((k, v) => true, false), "filter-02");
var filter = new ProcessorGraphNode <string, string>("filter-02", filterParameters);
source.AppendChild(filter);
nodes.Add(filter);
var to = new StreamSinkNode <string, string>(
new StaticTopicNameExtractor <string, string>("topic2"), "to-03",
new Stream.Internal.Produced <string, string>(
new StringSerDes(),
new StringSerDes())
);
filter.AppendChild(to);
nodes.Add(to);
builder.BuildTopology(root, nodes);
Assert.IsTrue(root.AllParentsWrittenToTopology);
Assert.IsTrue(source.AllParentsWrittenToTopology);
Assert.IsTrue(filter.AllParentsWrittenToTopology);
Assert.IsTrue(to.AllParentsWrittenToTopology);
var topology = builder.BuildTopology();
Assert.IsTrue(topology.SourceOperators.ContainsKey("topic"));
Assert.IsTrue(topology.ProcessorOperators.ContainsKey("filter-02"));
Assert.IsTrue(topology.SinkOperators.ContainsKey("topic2"));
}
public IKStream <K, VR> FlatMapValues <VR>(IValueMapperWithKey <K, V, IEnumerable <VR> > mapper, string named = null)
{
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, FLATMAPVALUES_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFlatMapValues <K, V, VR>(mapper), name);
ProcessorGraphNode <K, V> flatMapValuesNode = new ProcessorGraphNode <K, V>(name, processorParameters);
flatMapValuesNode.ValueChangingOperation = true;
builder.AddGraphNode(this.node, flatMapValuesNode);
// value serde cannot be preserved
return(new KStream <K, VR>(
name,
this.keySerdes,
null,
this.setSourceNodes,
flatMapValuesNode,
builder));
}
private IKGroupedStream <KR, V> DoGroup <KR>(IKeyValueMapper <K, V, KR> keySelector, Grouped <KR, V> grouped)
{
if (keySelector == null)
{
throw new ArgumentNullException("GroupBy() doesn't allow null selector function");
}
ProcessorGraphNode <K, V> selectKeyMapNode = InternalSelectKey(keySelector, grouped.Named);
selectKeyMapNode.KeyChangingOperation = true;
builder.AddGraphNode(this.node, selectKeyMapNode);
return(new KGroupedStream <KR, V>(
selectKeyMapNode.streamGraphNode,
grouped,
this.setSourceNodes,
selectKeyMapNode,
builder));
}
public IKStream <KR, V> SelectKey <KR>(IKeyValueMapper <K, V, KR> mapper, string named = null)
{
if (mapper == null)
{
throw new ArgumentNullException("SelectKey() doesn't allow null mapper function");
}
ProcessorGraphNode <K, V> selectKeyProcessorNode = InternalSelectKey(mapper, named);
selectKeyProcessorNode.KeyChangingOperation = true;
builder.AddGraphNode(node, selectKeyProcessorNode);
// key serde cannot be preserved
return(new KStream <KR, V>(
selectKeyProcessorNode.streamGraphNode,
null,
valueSerdes,
setSourceNodes,
selectKeyProcessorNode,
builder));
}
public IKStream <KR, VR> Map <KR, VR>(IKeyValueMapper <K, V, KeyValuePair <KR, VR> > mapper, string named = null)
{
if (mapper == null)
{
throw new ArgumentNullException($"Map() doesn't allow null mapper function");
}
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, MAP_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamMap <K, V, KR, VR>(mapper), name);
ProcessorGraphNode <K, V> mapProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
mapProcessorNode.KeyChangingOperation = true;
builder.AddGraphNode(node, mapProcessorNode);
// key and value serde cannot be preserved
return(new KStream <KR, VR>(
name,
null,
null,
setSourceNodes,
mapProcessorNode,
builder));
}
public KStream <K, VR> Join <K, V, V0, VR>(
KStream <K, V> joinLeft,
KStream <K, V0> joinRight,
IValueJoiner <V, V0, VR> joiner,
JoinWindowOptions windows,
StreamJoinProps <K, V, V0> joined)
{
var named = new Named(joined.Name);
var joinLeftSuffix = rightOuter ? "-outer-this-join" : "-this-join";
var joinRightSuffix = leftOuter ? "-outer-other-join" : "-other-join";
var leftWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME);
var rightWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME);
var joinLeftGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME);
var joinRightGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME);
var joinLeftName = named.SuffixWithOrElseGet(joinLeftSuffix, joinLeftGeneratedName);
var joinRightName = named.SuffixWithOrElseGet(joinRightSuffix, joinRightGeneratedName);
var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME);
StreamGraphNode leftStreamsGraphNode = joinLeft.Node;
StreamGraphNode rightStreamsGraphNode = joinRight.Node;
var userProvidedBaseStoreName = joined.StoreName;
var leftStoreSupplier = joined.LeftStoreSupplier;
var rightStoreSupplier = joined.RightStoreSupplier;
StoreBuilder <WindowStore <K, V> > leftWindowStore;
StoreBuilder <WindowStore <K, V0> > rightWindowStore;
AssertUniqueStoreNames(leftStoreSupplier, rightStoreSupplier);
if (leftStoreSupplier == null)
{
var thisJoinStoreName = userProvidedBaseStoreName == null ? joinLeftGeneratedName : userProvidedBaseStoreName + joinLeftSuffix;
leftWindowStore = JoinWindowStoreBuilder(thisJoinStoreName, windows, joined.KeySerdes, joined.LeftValueSerdes);
}
else
{
AssertWindowSettings(leftStoreSupplier, windows);
leftWindowStore = Stores.WindowStoreBuilder(leftStoreSupplier, joined.KeySerdes, joined.LeftValueSerdes);
}
if (rightStoreSupplier == null)
{
var otherJoinStoreName = userProvidedBaseStoreName == null ? joinRightGeneratedName : userProvidedBaseStoreName + joinRightSuffix;
rightWindowStore = JoinWindowStoreBuilder(otherJoinStoreName, windows, joined.KeySerdes, joined.RightValueSerdes);
}
else
{
AssertWindowSettings(rightStoreSupplier, windows);
rightWindowStore = Stores.WindowStoreBuilder(rightStoreSupplier, joined.KeySerdes, joined.RightValueSerdes);
}
var leftStream = new KStreamJoinWindow <K, V>(leftWindowStore.Name);
var leftStreamParams = new ProcessorParameters <K, V>(leftStream, leftWindowStreamProcessorName);
var leftNode = new ProcessorGraphNode <K, V>(leftWindowStreamProcessorName, leftStreamParams);
builder.AddGraphNode(leftStreamsGraphNode, leftNode);
var rightStream = new KStreamJoinWindow <K, V0>(rightWindowStore.Name);
var rightStreamParams = new ProcessorParameters <K, V0>(rightStream, rightWindowStreamProcessorName);
var rightNode = new ProcessorGraphNode <K, V0>(rightWindowStreamProcessorName, rightStreamParams);
builder.AddGraphNode(rightStreamsGraphNode, rightNode);
var joinL = new KStreamKStreamJoin <K, V, V0, VR>(joinLeftName, rightWindowStore.Name, windows.beforeMs, windows.afterMs, joiner, leftOuter);
var joinLParams = new ProcessorParameters <K, V>(joinL, joinLeftName);
var joinR = new KStreamKStreamJoin <K, V0, V, VR>(joinRightName, leftWindowStore.Name, windows.beforeMs, windows.afterMs, joiner.Reverse(), rightOuter);
var joinRParams = new ProcessorParameters <K, V0>(joinR, joinRightName);
var merge = new PassThrough <K, VR>();
var mergeParams = new ProcessorParameters <K, VR>(merge, joinMergeName);
var joinNode = new StreamStreamJoinNode <K, V, V0, VR>(
joinMergeName,
joiner,
joinLParams,
joinRParams,
mergeParams,
leftStreamParams,
rightStreamParams,
leftWindowStore,
rightWindowStore,
joined);
builder.AddGraphNode(new List <StreamGraphNode> {
leftStreamsGraphNode, rightStreamsGraphNode
}, joinNode);
ISet <string> allSourceNodes = new HashSet <string>(joinLeft.SetSourceNodes);
allSourceNodes.AddRange(joinRight.SetSourceNodes);
return(new KStream <K, VR>(
joinMergeName,
joined.KeySerdes,
null,
allSourceNodes.ToList(),
joinNode,
builder));
}
