private void GenerateCodeRefs(object parameter) { this.OnProcessing = true; Task.Factory.StartNew(() => { try { ProcessorParameters parameters = new ProcessorParameters { WithSqlProcedureIntegration = true, CompilationMode = this.SelectedCompilationMode.ToString(), }; _processor.Execute(new CodeRefProcessorBehavior(), parameters); } catch (Exception x) { this.WriteProcessLog(new ProcessMessageEventArgs(ProcessMessageType.Error, "CodeRefs Generation Error -> {0}", x.Message)); } this.OnProcessing = false; }, TaskCreationOptions.LongRunning); }
internal IGlobalKTable <K, V> GlobalTable <K, V>(string topic, ConsumedInternal <K, V> consumed, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized) { if (string.IsNullOrEmpty(topic)) { throw new ArgumentException("topic can't be null or empty", nameof(topic)); } // explicitly disable logging for global stores materialized.WithLoggingDisabled(); string sourceName = new Named(consumed.Named).SuffixWithOrElseGet(TABLE_SOURCE_SUFFIX, this, KStream.SOURCE_NAME); string tableSourceName = new Named(consumed.Named).OrElseGenerateWithPrefix(this, KTable.SOURCE_NAME); string storeName = materialized.StoreName; // enforce store name as queryable name to always materialize global table stores var tableSource = new KTableSource <K, V>(storeName, storeName); var processorParameters = new ProcessorParameters <K, V>(tableSource, tableSourceName); var tableSourceNode = new TableSourceNode <K, V, IKeyValueStore <Bytes, byte[]> >( topic, tableSourceName, sourceName, consumed, materialized, processorParameters, true); this.AddGraphNode(root, tableSourceNode); return(new GlobalKTable <K, V>(new KTableSourceValueGetterSupplier <K, V>(storeName), materialized.QueryableStoreName)); }
internal IKTable <K, V> Table <K, V>(string topic, ConsumedInternal <K, V> consumed, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized) { var sourceName = new Named(consumed.Named).SuffixWithOrElseGet(TABLE_SOURCE_SUFFIX, this, KStream.SOURCE_NAME); var tableSourceName = new Named(consumed.Named).OrElseGenerateWithPrefix(this, KTable.SOURCE_NAME); KTableSource <K, V> tableSource = new KTableSource <K, V>(materialized.StoreName, materialized.QueryableStoreName); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(tableSource, tableSourceName); var tableSourceNode = new TableSourceNode <K, V, IKeyValueStore <Bytes, byte[]> >( topic, tableSourceName, sourceName, consumed, materialized, processorParameters, false); this.AddGraphNode(root, tableSourceNode); return(new KTable <K, V, V>(tableSourceName, consumed.KeySerdes, consumed.ValueSerdes, new List <string> { sourceName }, materialized.QueryableStoreName, tableSource, tableSourceNode, this)); }
private IKStream <K, V>[] DoBranch(string named = null, params Func <K, V, bool>[] predicates) { var namedInternal = new Named(named); if (predicates.Length == 0) { throw new ArgumentException("branch() requires at least one predicate"); } String branchName = namedInternal.OrElseGenerateWithPrefix(this.builder, BRANCH_NAME); String[] childNames = new String[predicates.Length]; for (int i = 0; i < predicates.Length; i++) { childNames[i] = namedInternal.SuffixWithOrElseGet($"predicate-{i}", this.builder, BRANCHCHILD_NAME); } ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamBranch <K, V>(predicates, childNames), branchName); ProcessorGraphNode <K, V> branchNode = new ProcessorGraphNode <K, V>(branchName, processorParameters); this.builder.AddGraphNode(node, branchNode); IKStream <K, V>[] branchChildren = new IKStream <K, V> [predicates.Length]; for (int i = 0; i < predicates.Length; i++) { ProcessorParameters <K, V> innerProcessorParameters = new ProcessorParameters <K, V>(new PassThrough <K, V>(), childNames[i]); ProcessorGraphNode <K, V> branchChildNode = new ProcessorGraphNode <K, V>(childNames[i], innerProcessorParameters); builder.AddGraphNode(branchNode, branchChildNode); branchChildren[i] = new KStream <K, V>(childNames[i], this.keySerdes, this.valueSerdes, setSourceNodes, branchChildNode, builder); } return(branchChildren); }
public IKStream <K, VR> MapValues <VR>(IValueMapperWithKey <K, V, VR> mapper, string named = null) { if (mapper == null) { throw new ArgumentNullException($"Mapper function can't be null"); } String name = new Named(named).OrElseGenerateWithPrefix(this.builder, MAPVALUES_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamMapValues <K, V, VR>(mapper), name); ProcessorGraphNode <K, V> mapValuesProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters); mapValuesProcessorNode.ValueChangingOperation = true; builder.AddGraphNode(this.node, mapValuesProcessorNode); // value serde cannot be preserved return(new KStream <K, VR>( name, this.keySerdes, null, this.setSourceNodes, mapValuesProcessorNode, builder)); }
internal void AddGlobalStore <K, V, S>(string topicName, StoreBuilder <S> storeBuilder, string sourceName, ConsumedInternal <K, V> consumed, ProcessorParameters <K, V> processorParameters) where S : IStateStore { string processorName = processorParameters.ProcessorName; ValidateGlobalStoreArguments(sourceName, topicName, processorName, processorParameters.Processor, storeBuilder.Name, storeBuilder.LoggingEnabled); ValidateTopicNotAlreadyRegistered(topicName); var predecessors = new[] { sourceName }; var nodeFactory = new ProcessorNodeFactory <K, V>(processorName, predecessors, processorParameters.Processor); globalTopics.Add(topicName); nodeFactories.Add(sourceName, new SourceNodeFactory <K, V>(sourceName, topicName, consumed.TimestampExtractor, consumed.KeySerdes, consumed.ValueSerdes)); // TODO: ? // nodeToSourceTopics.put(sourceName, Arrays.asList(topics)); nodeGrouper.Add(sourceName); nodeFactory.AddStateStore(storeBuilder.Name); nodeFactories.Add(processorName, nodeFactory); nodeGrouper.Add(processorName); nodeGrouper.Unite(processorName, predecessors); globalStateBuilders.Add(storeBuilder.Name, storeBuilder); ConnectSourceStoreAndTopic(storeBuilder.Name, topicName); nodeGroups = null; }
public void Print(Printed <K, V> printed) { var name = new Named(printed.Name).OrElseGenerateWithPrefix(this.builder, PRINTING_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(printed.Build(this.nameNode), name); ProcessorGraphNode <K, V> printNode = new ProcessorGraphNode <K, V>(name, processorParameters); builder.AddGraphNode(node, printNode); }
public void Foreach(Action <K, V> action, string named = null) { String name = new Named(named).OrElseGenerateWithPrefix(this.builder, FOREACH_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, false), name); ProcessorGraphNode <K, V> foreachNode = new ProcessorGraphNode <K, V>(name, processorParameters); this.builder.AddGraphNode(node, foreachNode); }
private IKStream <K, V> DoFilter(Func <K, V, bool> predicate, string named, bool not) { string name = new Named(named).OrElseGenerateWithPrefix(this.builder, FILTER_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFilter <K, V>(predicate, not), name); ProcessorGraphNode <K, V> filterProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters); this.builder.AddGraphNode(node, filterProcessorNode); return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, filterProcessorNode, this.builder)); }
public TableSourceNode(string topicName, string streamGraphNode, string sourceName, ConsumedInternal <K, V> consumed, Materialized <K, V, S> materialized, ProcessorParameters <K, V> processorParameters, bool isGlobalKTable = false) : base(topicName, streamGraphNode, consumed) { this.materialized = materialized; this.processorParameters = processorParameters; this.sourceName = sourceName; this.isGlobalKTable = isGlobalKTable; }
public void Foreach(Action <K, V> action, string named = null) { if (action == null) { throw new ArgumentNullException("Foreach() doesn't allow null action function "); } String name = new Named(named).OrElseGenerateWithPrefix(this.builder, FOREACH_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, false), name); ProcessorGraphNode <K, V> foreachNode = new ProcessorGraphNode <K, V>(name, processorParameters); this.builder.AddGraphNode(node, foreachNode); }
public IKStream <KR, VR> FlatMap <KR, VR>(IKeyValueMapper <K, V, IEnumerable <KeyValuePair <KR, VR> > > mapper, string named = null) { var name = new Named(named).OrElseGenerateWithPrefix(this.builder, FLATMAP_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFlatMap <K, V, KR, VR>(mapper), name); ProcessorGraphNode <K, V> flatMapNode = new ProcessorGraphNode <K, V>(name, processorParameters); flatMapNode.KeyChangingOperation = true; builder.AddGraphNode(node, flatMapNode); // key and value serde cannot be preserved return(new KStream <KR, VR>(name, null, null, setSourceNodes, flatMapNode, builder)); }
private ProcessorGraphNode <K, V> InternalSelectKey <KR>(IKeyValueMapper <K, V, KR> mapper, string named = null) { var name = new Named(named).OrElseGenerateWithPrefix(this.builder, KEY_SELECT_NAME); WrappedKeyValueMapper <K, V, KeyValuePair <KR, V> > internalMapper = new WrappedKeyValueMapper <K, V, KeyValuePair <KR, V> >( (key, value) => new KeyValuePair <KR, V>(mapper.Apply(key, value), value)); KStreamMap <K, V, KR, V> kStreamMap = new KStreamMap <K, V, KR, V>(internalMapper); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(kStreamMap, name); return(new ProcessorGraphNode <K, V>(name, processorParameters)); }
public void Print(Printed <K, V> printed) { if (printed == null) { throw new ArgumentNullException("Print() doesn't allow null printed instance"); } var name = new Named(printed.Name).OrElseGenerateWithPrefix(this.builder, PRINTING_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(printed.Build(this.nameNode), name); ProcessorGraphNode <K, V> printNode = new ProcessorGraphNode <K, V>(name, processorParameters); builder.AddGraphNode(node, printNode); }
private IKStream <K, V> DoFilter(Func <K, V, bool> predicate, string named, bool not) { if (predicate == null) { throw new ArgumentNullException($"Filter() doesn't allow null predicate function"); } string name = new Named(named).OrElseGenerateWithPrefix(this.builder, FILTER_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFilter <K, V>(predicate, not), name); ProcessorGraphNode <K, V> filterProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters); this.builder.AddGraphNode(node, filterProcessorNode); return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, filterProcessorNode, this.builder)); }
public IKStream <K, V> ToStream(string named = null) { string name = new Named(named).OrElseGenerateWithPrefix(this.builder, TOSTREAM_NAME); var p = new WrapperValueMapperWithKey <K, Change <V>, V>((k, v) => v.NewValue); IProcessorSupplier <K, Change <V> > processorMapValues = new KStreamMapValues <K, Change <V>, V>(p); ProcessorParameters <K, Change <V> > processorParameters = new ProcessorParameters <K, Change <V> >(processorMapValues, name); ProcessorGraphNode <K, Change <V> > toStreamNode = new ProcessorGraphNode <K, Change <V> >(name, processorParameters); builder.AddGraphNode(this.node, toStreamNode); // we can inherit parent key and value serde return(new KStream <K, V>(name, this.keySerdes, this.valueSerdes, this.setSourceNodes, toStreamNode, builder)); }
public IKStream <K, V> Peek(Action <K, V> action, string named = null) { String name = new Named(named).OrElseGenerateWithPrefix(this.builder, PEEK_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamPeek <K, V>(action, true), name); ProcessorGraphNode <K, V> peekNode = new ProcessorGraphNode <K, V>(name, processorParameters); builder.AddGraphNode(node, peekNode); return(new KStream <K, V>( name, keySerdes, valueSerdes, setSourceNodes, peekNode, builder)); }
public void WriteTopologyTest() { var builder = new InternalTopologyBuilder(); List <StreamGraphNode> nodes = new List <StreamGraphNode>(); RootNode root = new RootNode(); var source = new StreamSourceNode <string, string>( "topic", "source-01", new Stream.Internal.ConsumedInternal <string, string>("source-01", new StringSerDes(), new StringSerDes(), null)); root.AppendChild(source); nodes.Add(source); var filterParameters = new ProcessorParameters <string, string>( new KStreamFilter <string, string>((k, v) => true, false), "filter-02"); var filter = new ProcessorGraphNode <string, string>("filter-02", filterParameters); source.AppendChild(filter); nodes.Add(filter); var to = new StreamSinkNode <string, string>( new StaticTopicNameExtractor <string, string>("topic2"), "to-03", new Stream.Internal.Produced <string, string>( new StringSerDes(), new StringSerDes()) ); filter.AppendChild(to); nodes.Add(to); builder.BuildTopology(root, nodes); Assert.IsTrue(root.AllParentsWrittenToTopology); Assert.IsTrue(source.AllParentsWrittenToTopology); Assert.IsTrue(filter.AllParentsWrittenToTopology); Assert.IsTrue(to.AllParentsWrittenToTopology); var topology = builder.BuildTopology(); Assert.IsTrue(topology.SourceOperators.ContainsKey("topic")); Assert.IsTrue(topology.ProcessorOperators.ContainsKey("filter-02")); Assert.IsTrue(topology.SinkOperators.ContainsKey("topic2")); }
public IKStream <K, VR> FlatMapValues <VR>(IValueMapperWithKey <K, V, IEnumerable <VR> > mapper, string named = null) { var name = new Named(named).OrElseGenerateWithPrefix(this.builder, FLATMAPVALUES_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamFlatMapValues <K, V, VR>(mapper), name); ProcessorGraphNode <K, V> flatMapValuesNode = new ProcessorGraphNode <K, V>(name, processorParameters); flatMapValuesNode.ValueChangingOperation = true; builder.AddGraphNode(this.node, flatMapValuesNode); // value serde cannot be preserved return(new KStream <K, VR>( name, this.keySerdes, null, this.setSourceNodes, flatMapValuesNode, builder)); }
private void Process(object parameter) { this.OnProcessing = true; Task.Factory.StartNew(() => { try { ProcessorParameters parameters = new ProcessorParameters { CompilationMode = this.SelectedCompilationMode.ToString(), TableNames = this.Tables.Where(t => t.IsChecked).Select(i => i.Item.Name), WithSqlProcedureIntegration = this.WithSqlProcedureIntegration }; _processor.Execute(new FullProcessorBehavior(), parameters); this.OnProcessing = false; if (_processor.GeneratedServices != null && _processor.GeneratedServices.Count != 0) { string message = "New Services has been generated and added to the solution.{0}{0}{1}{0}{0}Do not forget to register them if you use ServiceLocator, ServiceProxy or other IoC on client-side."; MessageBoxHelper.Show( string.Format(message, Environment.NewLine, string.Join(", ", _processor.GeneratedServices.OrderBy(i => i))), "Warning!", MessageBoxButton.OK, MessageBoxImage.Warning); } } catch (Exception x) { this.WriteProcessLog(new ProcessMessageEventArgs(ProcessMessageType.Error, "Processor Execution Error -> {0}", x.Message)); } }, TaskCreationOptions.LongRunning); }
public KTableKTableJoinNode( string name, ProcessorParameters <K, Change <V1> > joinLeftParams, ProcessorParameters <K, Change <V2> > joinRightParams, string leftJoinSideName, string rightJoinSideName, string[] leftStoreNames, string[] rightStoreNames, string queryableStoreName, StoreBuilder <TimestampedKeyValueStore <K, VR> > storeBuilder) : base(name, null, joinLeftParams, joinRightParams, null, leftJoinSideName, rightJoinSideName) { this.leftStoreNames = leftStoreNames; this.rightStoreNames = rightStoreNames; this.queryableStoreName = queryableStoreName; this.storeBuilder = storeBuilder; JoinMergeProcessorSupplier = new KTableKTableJoinMerger <K, V1, V2, VR>( (IKTableProcessorSupplier <K, V1, VR>)joinLeftParams.Processor, (IKTableProcessorSupplier <K, V2, VR>)joinRightParams.Processor, queryableStoreName); JoinMergeParams = new ProcessorParameters <K, Change <VR> >(JoinMergeProcessorSupplier, name); }
public IKStream <KR, VR> Map <KR, VR>(IKeyValueMapper <K, V, KeyValuePair <KR, VR> > mapper, string named = null) { if (mapper == null) { throw new ArgumentNullException($"Map() doesn't allow null mapper function"); } string name = new Named(named).OrElseGenerateWithPrefix(this.builder, MAP_NAME); ProcessorParameters <K, V> processorParameters = new ProcessorParameters <K, V>(new KStreamMap <K, V, KR, VR>(mapper), name); ProcessorGraphNode <K, V> mapProcessorNode = new ProcessorGraphNode <K, V>(name, processorParameters); mapProcessorNode.KeyChangingOperation = true; builder.AddGraphNode(node, mapProcessorNode); // key and value serde cannot be preserved return(new KStream <KR, VR>( name, null, null, setSourceNodes, mapProcessorNode, builder)); }
public KStream <K, VR> Join <K, V, V0, VR>( KStream <K, V> joinLeft, KStream <K, V0> joinRight, IValueJoiner <V, V0, VR> joiner, JoinWindowOptions windows, StreamJoinProps <K, V, V0> joined) { var named = new Named(joined.Name); var joinLeftSuffix = rightOuter ? "-outer-this-join" : "-this-join"; var joinRightSuffix = leftOuter ? "-outer-other-join" : "-other-join"; var leftWindowStreamProcessorName = named.SuffixWithOrElseGet("-this-windowed", builder, KStream.WINDOWED_NAME); var rightWindowStreamProcessorName = named.SuffixWithOrElseGet("-other-windowed", builder, KStream.WINDOWED_NAME); var joinLeftGeneratedName = rightOuter ? builder.NewProcessorName(KStream.OUTERTHIS_NAME) : builder.NewProcessorName(KStream.JOINTHIS_NAME); var joinRightGeneratedName = leftOuter ? builder.NewProcessorName(KStream.OUTEROTHER_NAME) : builder.NewProcessorName(KStream.JOINOTHER_NAME); var joinLeftName = named.SuffixWithOrElseGet(joinLeftSuffix, joinLeftGeneratedName); var joinRightName = named.SuffixWithOrElseGet(joinRightSuffix, joinRightGeneratedName); var joinMergeName = named.SuffixWithOrElseGet("-merge", builder, KStream.MERGE_NAME); StreamGraphNode leftStreamsGraphNode = joinLeft.Node; StreamGraphNode rightStreamsGraphNode = joinRight.Node; var userProvidedBaseStoreName = joined.StoreName; var leftStoreSupplier = joined.LeftStoreSupplier; var rightStoreSupplier = joined.RightStoreSupplier; StoreBuilder <WindowStore <K, V> > leftWindowStore; StoreBuilder <WindowStore <K, V0> > rightWindowStore; AssertUniqueStoreNames(leftStoreSupplier, rightStoreSupplier); if (leftStoreSupplier == null) { var thisJoinStoreName = userProvidedBaseStoreName == null ? joinLeftGeneratedName : userProvidedBaseStoreName + joinLeftSuffix; leftWindowStore = JoinWindowStoreBuilder(thisJoinStoreName, windows, joined.KeySerdes, joined.LeftValueSerdes); } else { AssertWindowSettings(leftStoreSupplier, windows); leftWindowStore = Stores.WindowStoreBuilder(leftStoreSupplier, joined.KeySerdes, joined.LeftValueSerdes); } if (rightStoreSupplier == null) { var otherJoinStoreName = userProvidedBaseStoreName == null ? joinRightGeneratedName : userProvidedBaseStoreName + joinRightSuffix; rightWindowStore = JoinWindowStoreBuilder(otherJoinStoreName, windows, joined.KeySerdes, joined.RightValueSerdes); } else { AssertWindowSettings(rightStoreSupplier, windows); rightWindowStore = Stores.WindowStoreBuilder(rightStoreSupplier, joined.KeySerdes, joined.RightValueSerdes); } var leftStream = new KStreamJoinWindow <K, V>(leftWindowStore.Name); var leftStreamParams = new ProcessorParameters <K, V>(leftStream, leftWindowStreamProcessorName); var leftNode = new ProcessorGraphNode <K, V>(leftWindowStreamProcessorName, leftStreamParams); builder.AddGraphNode(leftStreamsGraphNode, leftNode); var rightStream = new KStreamJoinWindow <K, V0>(rightWindowStore.Name); var rightStreamParams = new ProcessorParameters <K, V0>(rightStream, rightWindowStreamProcessorName); var rightNode = new ProcessorGraphNode <K, V0>(rightWindowStreamProcessorName, rightStreamParams); builder.AddGraphNode(rightStreamsGraphNode, rightNode); var joinL = new KStreamKStreamJoin <K, V, V0, VR>(joinLeftName, rightWindowStore.Name, windows.beforeMs, windows.afterMs, joiner, leftOuter); var joinLParams = new ProcessorParameters <K, V>(joinL, joinLeftName); var joinR = new KStreamKStreamJoin <K, V0, V, VR>(joinRightName, leftWindowStore.Name, windows.beforeMs, windows.afterMs, joiner.Reverse(), rightOuter); var joinRParams = new ProcessorParameters <K, V0>(joinR, joinRightName); var merge = new PassThrough <K, VR>(); var mergeParams = new ProcessorParameters <K, VR>(merge, joinMergeName); var joinNode = new StreamStreamJoinNode <K, V, V0, VR>( joinMergeName, joiner, joinLParams, joinRParams, mergeParams, leftStreamParams, rightStreamParams, leftWindowStore, rightWindowStore, joined); builder.AddGraphNode(new List <StreamGraphNode> { leftStreamsGraphNode, rightStreamsGraphNode }, joinNode); ISet <string> allSourceNodes = new HashSet <string>(joinLeft.SetSourceNodes); allSourceNodes.AddRange(joinRight.SetSourceNodes); return(new KStream <K, VR>( joinMergeName, joined.KeySerdes, null, allSourceNodes.ToList(), joinNode, builder)); }