private void GenerateCodeRefs(object parameter)
{
this.OnProcessing = true;
Task.Factory.StartNew(() =>
{
try
{
ProcessorParameters parameters = new ProcessorParameters
{
WithSqlProcedureIntegration = true,
CompilationMode = this.SelectedCompilationMode.ToString(),
};
_processor.Execute(new CodeRefProcessorBehavior(), parameters);
}
catch (Exception x)
{
this.WriteProcessLog(new ProcessMessageEventArgs(ProcessMessageType.Error, "CodeRefs Generation Error -> {0}", x.Message));
}
this.OnProcessing = false;
},
TaskCreationOptions.LongRunning);
}
internal IGlobalKTable <K, V> GlobalTable <K, V>(string topic, ConsumedInternal <K, V> consumed, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentException("topic can't be null or empty", nameof(topic));
}
// explicitly disable logging for global stores
materialized.WithLoggingDisabled();
string sourceName = new Named(consumed.Named).SuffixWithOrElseGet(TABLE_SOURCE_SUFFIX, this, KStream.SOURCE_NAME);
string tableSourceName = new Named(consumed.Named).OrElseGenerateWithPrefix(this, KTable.SOURCE_NAME);
string storeName = materialized.StoreName;
// enforce store name as queryable name to always materialize global table stores
var tableSource = new KTableSource <K, V>(storeName, storeName);
var processorParameters = new ProcessorParameters <K, V>(tableSource, tableSourceName);
var tableSourceNode = new TableSourceNode <K, V, IKeyValueStore <Bytes, byte[]> >(
topic, tableSourceName, sourceName, consumed,
materialized, processorParameters, true);
this.AddGraphNode(root, tableSourceNode);
return(new GlobalKTable <K, V>(new KTableSourceValueGetterSupplier <K, V>(storeName), materialized.QueryableStoreName));
}
internal IKTable <K, V> Table <K, V>(string topic, ConsumedInternal <K, V> consumed, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized)
{
var sourceName = new Named(consumed.Named).SuffixWithOrElseGet(TABLE_SOURCE_SUFFIX, this, KStream.SOURCE_NAME);
var tableSourceName = new Named(consumed.Named).OrElseGenerateWithPrefix(this, KTable.SOURCE_NAME);
KTableSource <K, V> tableSource = new KTableSource <K, V>(materialized.StoreName, materialized.QueryableStoreName);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(tableSource, tableSourceName);
var tableSourceNode = new TableSourceNode <K, V, IKeyValueStore <Bytes, byte[]> >(
topic, tableSourceName, sourceName, consumed,
materialized, processorParameters, false);
this.AddGraphNode(root, tableSourceNode);
return(new KTable <K, V, V>(tableSourceName,
consumed.KeySerdes,
consumed.ValueSerdes,
new List <string> {
sourceName
},
materialized.QueryableStoreName,
tableSource,
tableSourceNode,
this));
}
private IKStream <K, V>[] DoBranch(string named = null, params Func <K, V, bool>[] predicates)
{
var namedInternal = new Named(named);
if (predicates.Length == 0)
{
throw new ArgumentException("branch() requires at least one predicate");
}
String branchName = namedInternal.OrElseGenerateWithPrefix(this.builder, BRANCH_NAME);
String[] childNames = new String[predicates.Length];
for (int i = 0; i < predicates.Length; i++)
{
childNames[i] = namedInternal.SuffixWithOrElseGet($"predicate-{i}", this.builder, BRANCHCHILD_NAME);
}
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamBranch <K, V>(predicates, childNames), branchName);
ProcessorGraphNode <K, V> branchNode = new ProcessorGraphNode <K, V>(branchName, processorParameters);
this.builder.AddGraphNode(node, branchNode);
IKStream <K, V>[] branchChildren = new IKStream <K, V> [predicates.Length];
for (int i = 0; i < predicates.Length; i++)
{
ProcessorParameters <K, V> innerProcessorParameters = new ProcessorParameters <K, V>(new PassThrough <K, V>(), childNames[i]);
ProcessorGraphNode <K, V> branchChildNode = new ProcessorGraphNode <K, V>(childNames[i], innerProcessorParameters);
builder.AddGraphNode(branchNode, branchChildNode);
branchChildren[i] = new KStream <K, V>(childNames[i], this.keySerdes, this.valueSerdes, setSourceNodes, branchChildNode, builder);
}
return(branchChildren);
}
public IKStream <K, VR> MapValues <VR>(IValueMapperWithKey <K, V, VR> mapper, string named = null)
{
if (mapper == null)
{
throw new ArgumentNullException($"Mapper function can't be null");
}
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, MAPVALUES_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamMapValues <K, V, VR>(mapper), name);
ProcessorGraphNode <K, V> mapValuesProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
mapValuesProcessorNode.ValueChangingOperation = true;
builder.AddGraphNode(this.node, mapValuesProcessorNode);
// value serde cannot be preserved
return(new KStream <K, VR>(
name,
this.keySerdes,
null,
this.setSourceNodes,
mapValuesProcessorNode,
builder));
}
internal void AddGlobalStore <K, V, S>(string topicName,
StoreBuilder <S> storeBuilder,
string sourceName,
ConsumedInternal <K, V> consumed,
ProcessorParameters <K, V> processorParameters) where S : IStateStore
{
string processorName = processorParameters.ProcessorName;
ValidateGlobalStoreArguments(sourceName, topicName, processorName, processorParameters.Processor, storeBuilder.Name, storeBuilder.LoggingEnabled);
ValidateTopicNotAlreadyRegistered(topicName);
var predecessors = new[] { sourceName };
var nodeFactory = new ProcessorNodeFactory <K, V>(processorName, predecessors, processorParameters.Processor);
globalTopics.Add(topicName);
nodeFactories.Add(sourceName, new SourceNodeFactory <K, V>(sourceName, topicName, consumed.TimestampExtractor, consumed.KeySerdes, consumed.ValueSerdes));
// TODO: ?
// nodeToSourceTopics.put(sourceName, Arrays.asList(topics));
nodeGrouper.Add(sourceName);
nodeFactory.AddStateStore(storeBuilder.Name);
nodeFactories.Add(processorName, nodeFactory);
nodeGrouper.Add(processorName);
nodeGrouper.Unite(processorName, predecessors);
globalStateBuilders.Add(storeBuilder.Name, storeBuilder);
ConnectSourceStoreAndTopic(storeBuilder.Name, topicName);
nodeGroups = null;
}
public void Print(Printed <K, V> printed)
{
var name = new Named(printed.Name).OrElseGenerateWithPrefix(this.builder, PRINTING_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(printed.Build(this.nameNode), name);
ProcessorGraphNode <K, V> printNode = new ProcessorGraphNode <K, V>(name, processorParameters);
builder.AddGraphNode(node, printNode);
}
public void Foreach(Action <K, V> action, string named = null)
{
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, FOREACH_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, false), name);
ProcessorGraphNode <K, V> foreachNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, foreachNode);
}
private IKStream <K, V> DoFilter(Func <K, V, bool> predicate, string named, bool not)
{
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, FILTER_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFilter <K, V>(predicate, not), name);
ProcessorGraphNode <K, V> filterProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, filterProcessorNode);
return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, filterProcessorNode, this.builder));
}
public TableSourceNode(string topicName, string streamGraphNode,
string sourceName, ConsumedInternal <K, V> consumed,
Materialized <K, V, S> materialized, ProcessorParameters <K, V> processorParameters, bool isGlobalKTable = false)
: base(topicName, streamGraphNode, consumed)
{
this.materialized = materialized;
this.processorParameters = processorParameters;
this.sourceName = sourceName;
this.isGlobalKTable = isGlobalKTable;
}
public void Foreach(Action <K, V> action, string named = null)
{
if (action == null)
{
throw new ArgumentNullException("Foreach() doesn't allow null action function ");
}
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, FOREACH_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, false), name);
ProcessorGraphNode <K, V> foreachNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, foreachNode);
}
public IKStream <KR, VR> FlatMap <KR, VR>(IKeyValueMapper <K, V, IEnumerable <KeyValuePair <KR, VR> > > mapper, string named = null)
{
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, FLATMAP_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFlatMap <K, V, KR, VR>(mapper), name);
ProcessorGraphNode <K, V> flatMapNode = new ProcessorGraphNode <K, V>(name, processorParameters);
flatMapNode.KeyChangingOperation = true;
builder.AddGraphNode(node, flatMapNode);
// key and value serde cannot be preserved
return(new KStream <KR, VR>(name, null, null, setSourceNodes, flatMapNode, builder));
}
private ProcessorGraphNode <K, V> InternalSelectKey <KR>(IKeyValueMapper <K, V, KR> mapper, string named = null)
{
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, KEY_SELECT_NAME);
WrappedKeyValueMapper <K, V, KeyValuePair <KR, V> > internalMapper =
new WrappedKeyValueMapper <K, V, KeyValuePair <KR, V> >(
(key, value) => new KeyValuePair <KR, V>(mapper.Apply(key, value), value));
KStreamMap <K, V, KR, V> kStreamMap = new KStreamMap <K, V, KR, V>(internalMapper);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(kStreamMap, name);
return(new ProcessorGraphNode <K, V>(name, processorParameters));
}
public void Print(Printed <K, V> printed)
{
if (printed == null)
{
throw new ArgumentNullException("Print() doesn't allow null printed instance");
}
var name = new Named(printed.Name).OrElseGenerateWithPrefix(this.builder, PRINTING_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(printed.Build(this.nameNode), name);
ProcessorGraphNode <K, V> printNode = new ProcessorGraphNode <K, V>(name, processorParameters);
builder.AddGraphNode(node, printNode);
}
private IKStream <K, V> DoFilter(Func <K, V, bool> predicate, string named, bool not)
{
if (predicate == null)
{
throw new ArgumentNullException($"Filter() doesn't allow null predicate function");
}
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, FILTER_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFilter <K, V>(predicate, not), name);
ProcessorGraphNode <K, V> filterProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
this.builder.AddGraphNode(node, filterProcessorNode);
return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, filterProcessorNode, this.builder));
}
public IKStream <K, V> ToStream(string named = null)
{
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, TOSTREAM_NAME);
var p = new WrapperValueMapperWithKey <K, Change <V>, V>((k, v) => v.NewValue);
IProcessorSupplier <K, Change <V> > processorMapValues = new KStreamMapValues <K, Change <V>, V>(p);
ProcessorParameters <K, Change <V> > processorParameters = new ProcessorParameters <K, Change <V> >(processorMapValues, name);
ProcessorGraphNode <K, Change <V> > toStreamNode = new ProcessorGraphNode <K, Change <V> >(name, processorParameters);
builder.AddGraphNode(this.node, toStreamNode);
// we can inherit parent key and value serde
return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, toStreamNode, builder));
}
public IKStream <K, V> Peek(Action <K, V> action, string named = null)
{
String name = new Named(named).OrElseGenerateWithPrefix(this.builder, PEEK_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, true), name);
ProcessorGraphNode <K, V> peekNode = new ProcessorGraphNode <K, V>(name, processorParameters);
builder.AddGraphNode(node, peekNode);
return(new KStream <K, V>(
name,
keySerdes,
valueSerdes,
setSourceNodes,
peekNode,
builder));
}
public void WriteTopologyTest()
{
var builder = new InternalTopologyBuilder();
List <StreamGraphNode> nodes = new List <StreamGraphNode>();
RootNode root = new RootNode();
var source = new StreamSourceNode <string, string>(
"topic",
"source-01",
new Stream.Internal.ConsumedInternal <string, string>("source-01", new StringSerDes(),
new StringSerDes(), null));
root.AppendChild(source);
nodes.Add(source);
var filterParameters =
new ProcessorParameters <string, string>(
new KStreamFilter <string, string>((k, v) => true, false), "filter-02");
var filter = new ProcessorGraphNode <string, string>("filter-02", filterParameters);
source.AppendChild(filter);
nodes.Add(filter);
var to = new StreamSinkNode <string, string>(
new StaticTopicNameExtractor <string, string>("topic2"), "to-03",
new Stream.Internal.Produced <string, string>(
new StringSerDes(),
new StringSerDes())
);
filter.AppendChild(to);
nodes.Add(to);
builder.BuildTopology(root, nodes);
Assert.IsTrue(root.AllParentsWrittenToTopology);
Assert.IsTrue(source.AllParentsWrittenToTopology);
Assert.IsTrue(filter.AllParentsWrittenToTopology);
Assert.IsTrue(to.AllParentsWrittenToTopology);
var topology = builder.BuildTopology();
Assert.IsTrue(topology.SourceOperators.ContainsKey("topic"));
Assert.IsTrue(topology.ProcessorOperators.ContainsKey("filter-02"));
Assert.IsTrue(topology.SinkOperators.ContainsKey("topic2"));
}
public IKStream <K, VR> FlatMapValues <VR>(IValueMapperWithKey <K, V, IEnumerable <VR> > mapper, string named = null)
{
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, FLATMAPVALUES_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFlatMapValues <K, V, VR>(mapper), name);
ProcessorGraphNode <K, V> flatMapValuesNode = new ProcessorGraphNode <K, V>(name, processorParameters);
flatMapValuesNode.ValueChangingOperation = true;
builder.AddGraphNode(this.node, flatMapValuesNode);
// value serde cannot be preserved
return(new KStream <K, VR>(
name,
this.keySerdes,
null,
this.setSourceNodes,
flatMapValuesNode,
builder));
}
private void Process(object parameter)
{
this.OnProcessing = true;
Task.Factory.StartNew(() =>
{
try
{
ProcessorParameters parameters = new ProcessorParameters
{
CompilationMode = this.SelectedCompilationMode.ToString(),
TableNames = this.Tables.Where(t => t.IsChecked).Select(i => i.Item.Name),
WithSqlProcedureIntegration = this.WithSqlProcedureIntegration
};
_processor.Execute(new FullProcessorBehavior(), parameters);
this.OnProcessing = false;
if (_processor.GeneratedServices != null &&
_processor.GeneratedServices.Count != 0)
{
string message = "New Services has been generated and added to the solution.{0}{0}{1}{0}{0}Do not forget to register them if you use ServiceLocator, ServiceProxy or other IoC on client-side.";
MessageBoxHelper.Show(
string.Format(message, Environment.NewLine, string.Join(", ", _processor.GeneratedServices.OrderBy(i => i))),
"Warning!",
MessageBoxButton.OK,
MessageBoxImage.Warning);
}
}
catch (Exception x)
{
this.WriteProcessLog(new ProcessMessageEventArgs(ProcessMessageType.Error, "Processor Execution Error -> {0}", x.Message));
}
},
TaskCreationOptions.LongRunning);
}
public KTableKTableJoinNode(
string name,
ProcessorParameters <K, Change <V1> > joinLeftParams,
ProcessorParameters <K, Change <V2> > joinRightParams,
string leftJoinSideName,
string rightJoinSideName,
string[] leftStoreNames,
string[] rightStoreNames,
string queryableStoreName,
StoreBuilder <TimestampedKeyValueStore <K, VR> > storeBuilder)
: base(name, null, joinLeftParams, joinRightParams, null, leftJoinSideName, rightJoinSideName)
{
this.leftStoreNames = leftStoreNames;
this.rightStoreNames = rightStoreNames;
this.queryableStoreName = queryableStoreName;
this.storeBuilder = storeBuilder;
JoinMergeProcessorSupplier = new KTableKTableJoinMerger <K, V1, V2, VR>(
(IKTableProcessorSupplier <K, V1, VR>)joinLeftParams.Processor,
(IKTableProcessorSupplier <K, V2, VR>)joinRightParams.Processor,
queryableStoreName);
JoinMergeParams = new ProcessorParameters <K, Change <VR> >(JoinMergeProcessorSupplier, name);
}
public IKStream <KR, VR> Map <KR, VR>(IKeyValueMapper <K, V, KeyValuePair <KR, VR> > mapper, string named = null)
{
if (mapper == null)
{
throw new ArgumentNullException($"Map() doesn't allow null mapper function");
}
string name = new Named(named).OrElseGenerateWithPrefix(this.builder, MAP_NAME);
ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamMap <K, V, KR, VR>(mapper), name);
ProcessorGraphNode <K, V> mapProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters);
mapProcessorNode.KeyChangingOperation = true;
builder.AddGraphNode(node, mapProcessorNode);
// key and value serde cannot be preserved
return(new KStream <KR, VR>(
name,
null,
null,
setSourceNodes,
mapProcessorNode,
builder));
}
public KStream <K, VR> Join <K, V, V0, VR>(
KStream <K, V> joinLeft,
KStream <K, V0> joinRight,
IValueJoiner <V, V0, VR> joiner,
JoinWindowOptions windows,
StreamJoinProps <K, V, V0> joined)
{
var named = new Named(joined.Name);
var joinLeftSuffix = rightOuter ? "-outer-this-join" : "-this-join";
var joinRightSuffix = leftOuter ? "-outer-other-join" : "-other-join";
var leftWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME);
var rightWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME);
var joinLeftGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME);
var joinRightGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME);
var joinLeftName = named.SuffixWithOrElseGet(joinLeftSuffix, joinLeftGeneratedName);
var joinRightName = named.SuffixWithOrElseGet(joinRightSuffix, joinRightGeneratedName);
var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME);
StreamGraphNode leftStreamsGraphNode = joinLeft.Node;
StreamGraphNode rightStreamsGraphNode = joinRight.Node;
var userProvidedBaseStoreName = joined.StoreName;
var leftStoreSupplier = joined.LeftStoreSupplier;
var rightStoreSupplier = joined.RightStoreSupplier;
StoreBuilder <WindowStore <K, V> > leftWindowStore;
StoreBuilder <WindowStore <K, V0> > rightWindowStore;
AssertUniqueStoreNames(leftStoreSupplier, rightStoreSupplier);
if (leftStoreSupplier == null)
{
var thisJoinStoreName = userProvidedBaseStoreName == null ? joinLeftGeneratedName : userProvidedBaseStoreName + joinLeftSuffix;
leftWindowStore = JoinWindowStoreBuilder(thisJoinStoreName, windows, joined.KeySerdes, joined.LeftValueSerdes);
}
else
{
AssertWindowSettings(leftStoreSupplier, windows);
leftWindowStore = Stores.WindowStoreBuilder(leftStoreSupplier, joined.KeySerdes, joined.LeftValueSerdes);
}
if (rightStoreSupplier == null)
{
var otherJoinStoreName = userProvidedBaseStoreName == null ? joinRightGeneratedName : userProvidedBaseStoreName + joinRightSuffix;
rightWindowStore = JoinWindowStoreBuilder(otherJoinStoreName, windows, joined.KeySerdes, joined.RightValueSerdes);
}
else
{
AssertWindowSettings(rightStoreSupplier, windows);
rightWindowStore = Stores.WindowStoreBuilder(rightStoreSupplier, joined.KeySerdes, joined.RightValueSerdes);
}
var leftStream = new KStreamJoinWindow <K, V>(leftWindowStore.Name);
var leftStreamParams = new ProcessorParameters <K, V>(leftStream, leftWindowStreamProcessorName);
var leftNode = new ProcessorGraphNode <K, V>(leftWindowStreamProcessorName, leftStreamParams);
builder.AddGraphNode(leftStreamsGraphNode, leftNode);
var rightStream = new KStreamJoinWindow <K, V0>(rightWindowStore.Name);
var rightStreamParams = new ProcessorParameters <K, V0>(rightStream, rightWindowStreamProcessorName);
var rightNode = new ProcessorGraphNode <K, V0>(rightWindowStreamProcessorName, rightStreamParams);
builder.AddGraphNode(rightStreamsGraphNode, rightNode);
var joinL = new KStreamKStreamJoin <K, V, V0, VR>(joinLeftName, rightWindowStore.Name, windows.beforeMs, windows.afterMs, joiner, leftOuter);
var joinLParams = new ProcessorParameters <K, V>(joinL, joinLeftName);
var joinR = new KStreamKStreamJoin <K, V0, V, VR>(joinRightName, leftWindowStore.Name, windows.beforeMs, windows.afterMs, joiner.Reverse(), rightOuter);
var joinRParams = new ProcessorParameters <K, V0>(joinR, joinRightName);
var merge = new PassThrough <K, VR>();
var mergeParams = new ProcessorParameters <K, VR>(merge, joinMergeName);
var joinNode = new StreamStreamJoinNode <K, V, V0, VR>(
joinMergeName,
joiner,
joinLParams,
joinRParams,
mergeParams,
leftStreamParams,
rightStreamParams,
leftWindowStore,
rightWindowStore,
joined);
builder.AddGraphNode(new List <StreamGraphNode> {
leftStreamsGraphNode, rightStreamsGraphNode
}, joinNode);
ISet <string> allSourceNodes = new HashSet <string>(joinLeft.SetSourceNodes);
allSourceNodes.AddRange(joinRight.SetSourceNodes);
return(new KStream <K, VR>(
joinMergeName,
joined.KeySerdes,
null,
allSourceNodes.ToList(),
joinNode,
builder));
}
