public static IFluentProcessMutatorBuilder CustomCode(this IFluentProcessMutatorBuilder builder, string name, CustomMutatorDelegate action) { return(builder.AddMutator(new CustomMutator(builder.ProcessBuilder.Result.Context) { Action = action, })); }
public static IFluentProcessMutatorBuilder ReplaceEmptyStringWithNull(this IFluentProcessMutatorBuilder builder, params string[] columns) { return(builder.AddMutator(new ReplaceEmptyStringWithNullMutator(builder.ProcessBuilder.Result.Context) { Columns = columns, })); }
public static IFluentProcessMutatorBuilder ProcessOnMultipleThreads(this IFluentProcessMutatorBuilder builder, ITopic topic, int threadCount, Action <int, IFluentProcessMutatorBuilder> mutatorBuilder) { var splitter = new Splitter <DefaultRowQueue>(topic, "ParallelSplitter") { InputProcess = builder.ProcessBuilder.Result, }; var merger = new ParallelMerger(topic, "ParallelMerger") { ProcessList = new List <IEvaluable>(), }; for (var i = 0; i < threadCount; i++) { var subBuilder = ProcessBuilder.Fluent; var subMutatorBuilder = subBuilder.ReadFrom(splitter); mutatorBuilder.Invoke(i, subMutatorBuilder); var subProcess = subBuilder.Result; merger.ProcessList.Add(subProcess); } builder.ProcessBuilder.Result = merger; return(builder); }
public static IFluentProcessMutatorBuilder RemoveAllRow(this IFluentProcessMutatorBuilder builder) { return(builder.AddMutator(new RemoveRowMutator(builder.ProcessBuilder.Result.Context) { Name = nameof(RemoveAllRow), })); }
public static IFluentProcessMutatorBuilder CreateTaggedVersions(this IFluentProcessMutatorBuilder builder, params object[] tags) { return(builder.AddMutator(new MultiplyByTagsMutator(builder.ProcessBuilder.Result.Context) { Tags = tags, })); }
public static IFluentProcessMutatorBuilder TrimString(this IFluentProcessMutatorBuilder builder, params string[] columns) { return(builder.AddMutator(new TrimStringMutator(builder.ProcessBuilder.Result.Context) { Columns = columns, })); }
public static IFluentProcessMutatorBuilder Explode(this IFluentProcessMutatorBuilder builder, string name, ExplodeDelegate rowCreator) { return(builder.AddMutator(new ExplodeMutator(builder.ProcessBuilder.Result.Context) { Name = name, RowCreator = rowCreator, })); }
/// <summary> /// Keeps only the first row of each key, and discard all subsequent rows with existing keys. /// <para>- input can be unordered</para> /// <para>- if a more refined logic is required to decide which row should be kept of rows with same key then <see cref="ReduceGroupToSingleRowMutatorFluent.ReduceGroupToSingleRow(IFluentProcessMutatorBuilder, ReduceGroupToSingleRowMutator)"/> or <see cref="SortedReduceGroupToSingleRowMutatorFluent.ReduceGroupToSingleRowOrdered(IFluentProcessMutatorBuilder, SortedReduceGroupToSingleRowMutator)"/></para> can be used instead. /// <para>- all keys are stored in memory</para> /// </summary> public static IFluentProcessMutatorBuilder RemoveDuplicateRows(this IFluentProcessMutatorBuilder builder, string name, Func <IReadOnlyRow, string> keyGenerator) { return(builder.AddMutator(new RemoveDuplicateRowsMutator(builder.ProcessBuilder.Result.Context) { Name = name, KeyGenerator = keyGenerator, })); }
public static IFluentProcessMutatorBuilder SortInMemory(this IFluentProcessMutatorBuilder builder, string name, Func <IEnumerable <IRow>, IEnumerable <IRow> > sorter) { return(builder.AddMutator(new MemorySortMutator(builder.ProcessBuilder.Result.Context) { Name = name, Sorter = sorter, })); }
public static IFluentProcessMutatorBuilder RemoveRow(this IFluentProcessMutatorBuilder builder, string name, RowTestDelegate rowTestDelegate) { return(builder.AddMutator(new RemoveRowMutator(builder.ProcessBuilder.Result.Context) { Name = name, RowFilter = rowTestDelegate, })); }
public static IProducer BuildToInMemoryRowCache(this IFluentProcessMutatorBuilder builder, string name = null) { return(new InMemoryRowCache(builder.ProcessBuilder.Result.Context) { Name = name, InputProcess = builder.Build(), }); }
public static IFluentProcessMutatorBuilder CustomCode(this IFluentProcessMutatorBuilder builder, string name, Action <IRow> action) { return(builder.AddMutator(new CustomMutator(builder.ProcessBuilder.Result.Context) { Action = row => { action.Invoke(row); return true; } })); }
public static IFluentProcessMutatorBuilder CreateTaggedVersions(this IFluentProcessMutatorBuilder builder, MultiplyByTagsMutator mutator) { return(builder.AddMutator(mutator)); }
public static IFluentProcessMutatorBuilder ThrowExceptionOnRowError(this IFluentProcessMutatorBuilder builder, ThrowExceptionOnRowErrorMutator mutator) { return(builder.AddMutators(mutator)); }
public static IFluentProcessMutatorBuilder SetTag(this IFluentProcessMutatorBuilder builder, SetTagMutator mutator) { return(builder.AddMutator(mutator)); }
public static IFluentProcessMutatorBuilder RemoveColumn(this IFluentProcessMutatorBuilder builder, RemoveColumnMutator mutator) { return(builder.AddMutators(mutator)); }
public static IFluentProcessMutatorBuilder ReplaceEmptyStringWithNull(this IFluentProcessMutatorBuilder builder, ReplaceEmptyStringWithNullMutator mutator) { return(builder.AddMutator(mutator)); }
public static IFluentProcessMutatorBuilder ReplaceNullWithValue(this IFluentProcessMutatorBuilder builder, TrimStringMutator mutator) { return(builder.AddMutators(mutator)); }
public static IFluentProcessMutatorBuilder MergeStringColumns(this IFluentProcessMutatorBuilder builder, MergeStringColumnsMutator mutator) { return(builder.AddMutator(mutator)); }
public static IFluentProcessMutatorBuilder RemoveRow(this IFluentProcessMutatorBuilder builder, RemoveRowWithErrorMutator mutator) { return(builder.AddMutator(mutator)); }
/// <summary> /// Keeps only the first row of each key, and discard all subsequent rows with existing keys. /// <para>- input can be unordered</para> /// <para>- if a more refined logic is required to decide which row should be kept of rows with same key then <see cref="ReduceGroupToSingleRowMutatorFluent.ReduceGroupToSingleRow(IFluentProcessMutatorBuilder, ReduceGroupToSingleRowMutator)"/> or <see cref="SortedReduceGroupToSingleRowMutatorFluent.ReduceGroupToSingleRowOrdered(IFluentProcessMutatorBuilder, SortedReduceGroupToSingleRowMutator)"/></para> can be used instead. /// <para>- all keys are stored in memory</para> /// </summary> public static IFluentProcessMutatorBuilder RemoveDuplicateRows(this IFluentProcessMutatorBuilder builder, RemoveDuplicateRowsMutator mutator) { return(builder.AddMutators(mutator)); }
public static IFluentProcessMutatorBuilder TrimString(this IFluentProcessMutatorBuilder builder, TrimStringMutator mutator) { return(builder.AddMutator(mutator)); }
public static IFluentProcessMutatorBuilder SortInMemory(this IFluentProcessMutatorBuilder builder, MemorySortMutator mutator) { return(builder.AddMutator(mutator)); }
/// <summary> /// <para>- input can be unordered</para> /// <para>- returns all aggregates at once when everything is processed (blocks execution)</para> /// <para>- memory footprint is high because all rows in all groups are collected before aggregation</para> /// <para>- if the input is ordered then <see cref="SortedMemoryAggregationMutatorFluent.AggregateOrdered(IFluentProcessMutatorBuilder, SortedMemoryAggregationMutator)"/> should be used for much lower memory footprint and stream-like behavior</para> /// <para>- if the input is unordered but only basic operations are used then <see cref="ContinuousAggregationMutatorFluent.AggregateContinuously(IFluentProcessMutatorBuilder, ContinuousAggregationMutator)"/> should be used</para> /// </summary> public static IFluentProcessMutatorBuilder Aggregate(this IFluentProcessMutatorBuilder builder, MemoryAggregationMutator mutator) { return(builder.AddMutators(mutator)); }
public static IFluentProcessMutatorBuilder ThrowExceptionOnRowError(this IFluentProcessMutatorBuilder builder) { return(builder.AddMutator(new ThrowExceptionOnRowErrorMutator(builder.ProcessBuilder.Result.Context))); }
/// <summary> /// <para>- input must be ordered by group key</para> /// <para>- returns each aggregate right after a group is processed (stream-like behavior like most mutators)</para> /// <para>- if there is an ordering mismatch in the input then later appearances of a previously processed key will create new aggregated group(s)</para> /// <para>- memory footprint is very low because only rows of one group are collected before aggregation is executed on them</para> /// </summary> public static IFluentProcessMutatorBuilder AggregateOrdered(this IFluentProcessMutatorBuilder builder, SortedMemoryAggregationMutator mutator) { return(builder.AddMutator(mutator)); }
public static IFluentProcessMutatorBuilder CustomCode(this IFluentProcessMutatorBuilder builder, CustomMutator mutator) { return(builder.AddMutators(mutator)); }
/// <summary> /// Throw an exception if a subsequent occurence of a row key is found. /// <para>- input can be unordered</para> /// <para>- all keys are stored in memory</para> /// </summary> public static IFluentProcessMutatorBuilder ThrowExceptionOnDuplicateKey(this IFluentProcessMutatorBuilder builder, ThrowExceptionOnDuplicateKeyMutator mutator) { return(builder.AddMutator(mutator)); }
public static IFluentProcessMutatorBuilder Explode(this IFluentProcessMutatorBuilder builder, ExplodeMutator mutator) { return(builder.AddMutators(mutator)); }