public IKTable <Windowed <K>, VR> Aggregate <VR>(Initializer <VR> initializer, Aggregator <K, V, VR> aggregator, Materialized <K, VR, WindowStore <Bytes, byte[]> > materialized, string named = null)
{
CheckIfParamNull(initializer, "initializer");
CheckIfParamNull(aggregator, "aggregator");
materialized = materialized ?? Materialized <K, VR, WindowStore <Bytes, byte[]> > .Create();
if (materialized.KeySerdes == null)
{
materialized.WithKeySerdes(KeySerdes);
}
string name = new Named(named).OrElseGenerateWithPrefix(builder, KGroupedStream.AGGREGATE_NAME);
materialized.UseProvider(builder, KGroupedStream.AGGREGATE_NAME);
var aggSupplier = new KStreamWindowAggregate <K, V, VR, W>(
windowOptions,
materialized.StoreName,
initializer,
aggregator);
ISerDes <Windowed <K> > windowSerdes = materialized.KeySerdes != null ? new TimeWindowedSerDes <K>(materialized.KeySerdes, windowOptions.Size) : null;
return(aggBuilder.BuildWindow(name,
new TimestampedWindowStoreMaterializer <K, VR, W>(windowOptions, materialized).Materialize(),
aggSupplier,
materialized.QueryableStoreName,
windowSerdes,
materialized.ValueSerdes));
}
/// <summary>
/// Create a <see cref="IGlobalKTable{K,V}"/> for the specified topic.
/// Input keyvalue records with <code>null</code> key will be dropped.
/// The resulting <see cref="IGlobalKTable{K,V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// However, no internal changelog topic is created since the original input topic can be used for recovery.
/// Note that <see cref="IGlobalKTable{K,V}"/> always applies <code>"auto.offset.reset"</code> strategy <code>"earliest"</code>
/// regardless of the specified value in <see cref="IStreamConfig"/>.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IGlobalKTable{K,V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IGlobalKTable <K, V> GlobalTable <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor)
{
materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create();
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
materialized.UseProvider(internalStreamBuilder, $"{topic}-")?.InitConsumed(consumedInternal);
return(internalStreamBuilder.GlobalTable(topic, consumedInternal, materialized));
}
/// <summary>
/// Create a <see cref="IKTable{K, V}"/> for the specified topic.
/// Input keyvalue records with null key will be dropped.
///
/// Note that the specified input topic must be partitioned by key.
/// If this is not the case the returned <see cref="IKTable{K, V}"/> will be corrupted.
///
/// The resulting <see cref="IKTable{K, V}"/> will be materialized in a local <see cref="IKeyValueStore{K, V}"/> using the given
/// <see cref="Materialized{K, V, S}"/> instance.
/// </summary>
/// <typeparam name="K">Key type of record</typeparam>
/// <typeparam name="V">Value type of record</typeparam>
/// <param name="topic">the topic name, can't be null</param>
/// <param name="keySerdes">Key deserializer</param>
/// <param name="valueSerdes">Value deserializer</param>
/// <param name="materialized">the instance of <see cref="Materialized{K, V, S}"/> used to materialize a state store.</param>
/// <param name="named">Processor name</param>
/// <param name="extractor">the timestamp extractor to be used. If null the default timestamp extractor from config will be used</param>
/// <returns>a <see cref="IKTable{K, V}"/> for the specified topic</returns>
/// <exception cref="ArgumentException">Throw <see cref="ArgumentException"/> if topic is null or empty</exception>
public IKTable <K, V> Table <K, V>(string topic, ISerDes <K> keySerdes, ISerDes <V> valueSerdes, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materialized, string named, ITimestampExtractor extractor)
{
if (string.IsNullOrEmpty(topic))
{
throw new ArgumentException("Topic of KTable must not be null or empty");
}
materialized = materialized ?? Materialized <K, V, IKeyValueStore <Bytes, byte[]> > .Create();
var consumedInternal = new ConsumedInternal <K, V>(named, keySerdes, valueSerdes, extractor);
materialized.UseProvider(internalStreamBuilder, $"{topic}-").InitConsumed(consumedInternal);
return(internalStreamBuilder.Table(topic, consumedInternal, materialized));
}
public IKTable <Windowed <K>, V> Reduce(Reducer <V> reducer, Materialized <K, V, WindowStore <Bytes, byte[]> > materialized, string named = null)
{
CheckIfParamNull(reducer, "reducer");
materialized = materialized ?? Materialized <K, V, WindowStore <Bytes, byte[]> > .Create();
if (materialized.KeySerdes == null)
{
materialized.WithKeySerdes(KeySerdes);
}
if (materialized.ValueSerdes == null)
{
materialized.WithValueSerdes(ValueSerdes);
}
string name = new Named(named).OrElseGenerateWithPrefix(builder, KGroupedStream.REDUCE_NAME);
materialized.UseProvider(builder, KGroupedStream.REDUCE_NAME);
var aggSupplier = new KStreamWindowAggregate <K, V, V, W>(
windowOptions,
materialized.StoreName,
() => default,
public IKTable <K, VR> Join <K, V, V0, VR>(
IKTable <K, V> tableLeft,
IKTable <K, V0> tableRight,
IValueJoiner <V, V0, VR> joiner,
string named,
Materialized <K, VR, IKeyValueStore <Bytes, byte[]> > materializedInternal)
{
var renamed = new Named(named);
var joinMergeName = renamed.OrElseGenerateWithPrefix(builder, KTable.MERGE_NAME);
ISet <string> allSourceNodes = new HashSet <string>((tableLeft as AbstractStream <K, V>).SetSourceNodes);
allSourceNodes.AddRange((tableRight as AbstractStream <K, V0>).SetSourceNodes);
materializedInternal.UseProvider(builder, $"{joinMergeName}-");
if (leftOuter)
{
(tableLeft as IKTableGetter <K, V>)?.EnableSendingOldValues();
}
if (rightOuter)
{
(tableRight as IKTableGetter <K, V0>)?.EnableSendingOldValues();
}
AbstractKTableKTableJoin <K, VR, V, V0> joinLeft = null;
AbstractKTableKTableJoin <K, VR, V0, V> joinRight = null;
if (!leftOuter) // INNER JOIN
{
joinLeft = new KTableKTableInnerJoin <K, VR, V, V0>((tableLeft as IKTableGetter <K, V>), (tableRight as IKTableGetter <K, V0>), joiner);
joinRight = new KTableKTableInnerJoin <K, VR, V0, V>((tableRight as IKTableGetter <K, V0>), (tableLeft as IKTableGetter <K, V>), joiner.Reverse());
}
else if (!rightOuter) // LEFT JOIN
{
joinLeft = new KTableKTableLeftJoin <K, VR, V, V0>((tableLeft as IKTableGetter <K, V>), (tableRight as IKTableGetter <K, V0>), joiner);
joinRight = new KTableKTableRightJoin <K, VR, V0, V>((tableRight as IKTableGetter <K, V0>), (tableLeft as IKTableGetter <K, V>), joiner.Reverse());
}
else // OUTER JOIN
{
joinLeft = new KTableKTableOuterJoin <K, VR, V, V0>((tableLeft as IKTableGetter <K, V>), (tableRight as IKTableGetter <K, V0>), joiner);
joinRight = new KTableKTableOuterJoin <K, VR, V0, V>((tableRight as IKTableGetter <K, V0>), (tableLeft as IKTableGetter <K, V>), joiner.Reverse());
}
var joinLeftName = renamed.SuffixWithOrElseGet("-join-this", builder, KTable.JOINTHIS_NAME);
var joinRigthName = renamed.SuffixWithOrElseGet("-join-other", builder, KTable.JOINOTHER_NAME);
var joinLeftProcessorParameters = new TableProcessorParameters <K, V>(joinLeft, joinLeftName);
var joinRightProcessorParameters = new TableProcessorParameters <K, V0>(joinRight, joinRigthName);
if (materializedInternal.KeySerdes == null && tableLeft is AbstractStream <K, V> )
{
materializedInternal.WithKeySerdes((tableLeft as AbstractStream <K, V>).KeySerdes);
}
ISerDes <K> keySerdes = materializedInternal.KeySerdes;
ISerDes <VR> ValueSerdes = materializedInternal.ValueSerdes;
string queryableStoreName = materializedInternal.QueryableStoreName;
StoreBuilder <TimestampedKeyValueStore <K, VR> > storeBuilder = queryableStoreName != null ? new TimestampedKeyValueStoreMaterializer <K, VR>(materializedInternal).Materialize() : null;
var tableNode = new KTableKTableJoinNode <K, V, V0, VR>(
joinMergeName,
joinLeftProcessorParameters,
joinRightProcessorParameters,
(tableLeft as AbstractStream <K, V>).NameNode,
(tableRight as AbstractStream <K, V0>).NameNode,
(tableLeft as IKTableGetter <K, V>).ValueGetterSupplier.StoreNames,
(tableRight as IKTableGetter <K, V0>).ValueGetterSupplier.StoreNames,
queryableStoreName,
storeBuilder);
builder.AddGraphNode((tableLeft as AbstractStream <K, V>).Node, tableNode);
return(new KTable <K, VR, VR>(
tableNode.streamGraphNode,
keySerdes,
ValueSerdes,
allSourceNodes.ToList(),
queryableStoreName,
tableNode.JoinMergeProcessorSupplier,
tableNode,
builder));
}
private IKTable <K, VR> DoMapValues <VR>(IValueMapperWithKey <K, V, VR> mapper, string named, Materialized <K, VR, IKeyValueStore <Bytes, byte[]> > materializedInternal)
{
if (mapper == null)
{
throw new ArgumentNullException($"MapValues() doesn't allow null mapper function");
}
ISerDes <K> keySerde;
ISerDes <VR> valueSerde;
String queryableStoreName;
StoreBuilder <TimestampedKeyValueStore <K, VR> > storeBuilder;
if (materializedInternal != null)
{
// we actually do not need to generate store names at all since if it is not specified, we will not
// materialize the store; but we still need to burn one index BEFORE generating the processor to keep compatibility.
if (materializedInternal.StoreName == null)
{
builder.NewStoreName(MAPVALUES_NAME);
}
keySerde = materializedInternal.KeySerdes != null ? materializedInternal.KeySerdes : this.keySerdes;
valueSerde = materializedInternal.ValueSerdes != null ? materializedInternal.ValueSerdes : null;
// ONLY FOR CALCULATE PROPERTY queriable
materializedInternal.UseProvider(null, null);
queryableStoreName = materializedInternal.QueryableStoreName;
// only materialize if materialized is specified and it has queryable name
storeBuilder = queryableStoreName != null ? new TimestampedKeyValueStoreMaterializer <K, VR>(materializedInternal).Materialize() : null;
}
else
{
keySerde = this.keySerdes;
valueSerde = null;
queryableStoreName = null;
storeBuilder = null;
}
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, MAPVALUES_NAME);
var processorSupplier = new KTableMapValues <K, V, VR>(this, mapper, queryableStoreName);
var processorParameters = new TableProcessorParameters <K, V>(processorSupplier, name);
var tableNode = new TableProcessorNode <K, V, K, VR>(
name,
processorParameters,
storeBuilder
);
builder.AddGraphNode(this.node, tableNode);
// don't inherit parent value serde, since this operation may change the value type, more specifically:
// we preserve the key following the order of 1) materialized, 2) parent, 3) null
// we preserve the value following the order of 1) materialized, 2) null
return(new KTable <K, V, VR>(
name,
keySerde,
valueSerde,
this.setSourceNodes,
queryableStoreName,
processorSupplier,
tableNode,
builder
));
}
private IKTable <K, V> DoFilter(Func <K, V, bool> predicate, string named, Materialized <K, V, IKeyValueStore <Bytes, byte[]> > materializedInternal, bool filterNot)
{
ISerDes <K> keySerde;
ISerDes <V> valueSerde;
String queryableStoreName;
StoreBuilder <TimestampedKeyValueStore <K, V> > storeBuilder;
if (predicate == null)
{
throw new ArgumentNullException($"Filter() doesn't allow null predicate function");
}
if (materializedInternal != null)
{
// we actually do not need to generate store names at all since if it is not specified, we will not
// materialize the store; but we still need to burn one index BEFORE generating the processor to keep compatibility.
if (materializedInternal.StoreName == null)
{
builder.NewStoreName(FILTER_NAME);
}
// we can inherit parent key and value serde if user do not provide specific overrides, more specifically:
// we preserve the key following the order of 1) materialized, 2) parent
keySerde = materializedInternal.KeySerdes != null ? materializedInternal.KeySerdes : this.keySerdes;
// we preserve the value following the order of 1) materialized, 2) parent
valueSerde = materializedInternal.ValueSerdes != null ? materializedInternal.ValueSerdes : this.valueSerdes;
// ONLY FOR CALCULATE PROPERTY queriable
materializedInternal.UseProvider(null, null);
queryableStoreName = materializedInternal.QueryableStoreName;
// only materialize if materialized is specified and it has queryable name
storeBuilder = queryableStoreName != null ? (new TimestampedKeyValueStoreMaterializer <K, V>(materializedInternal)).Materialize() : null;
}
else
{
keySerde = this.keySerdes;
valueSerde = this.valueSerdes;
queryableStoreName = null;
storeBuilder = null;
}
var name = new Named(named).OrElseGenerateWithPrefix(this.builder, FILTER_NAME);
IProcessorSupplier <K, Change <V> > processorSupplier = new KTableFilter <K, V>(this, predicate, filterNot, queryableStoreName);
var processorParameters = new TableProcessorParameters <K, V>(processorSupplier, name);
var tableNode = new TableProcessorNode <K, V, K, V>(
name,
processorParameters,
storeBuilder
);
builder.AddGraphNode(this.node, tableNode);
return(new KTable <K, V, V>(name,
keySerde,
valueSerde,
this.setSourceNodes,
queryableStoreName,
processorSupplier,
tableNode,
builder));
}
