public static IFluentProcessMutatorBuilder CustomCode(this IFluentProcessMutatorBuilder builder, string name, CustomMutatorDelegate action)
{
return(builder.AddMutator(new CustomMutator(builder.ProcessBuilder.Result.Context)
{
Action = action,
}));
}
public static IFluentProcessMutatorBuilder ReplaceEmptyStringWithNull(this IFluentProcessMutatorBuilder builder, params string[] columns)
{
return(builder.AddMutator(new ReplaceEmptyStringWithNullMutator(builder.ProcessBuilder.Result.Context)
{
Columns = columns,
}));
}
public static IFluentProcessMutatorBuilder ProcessOnMultipleThreads(this IFluentProcessMutatorBuilder builder, ITopic topic, int threadCount, Action <int, IFluentProcessMutatorBuilder> mutatorBuilder)
{
var splitter = new Splitter <DefaultRowQueue>(topic, "ParallelSplitter")
{
InputProcess = builder.ProcessBuilder.Result,
};
var merger = new ParallelMerger(topic, "ParallelMerger")
{
ProcessList = new List <IEvaluable>(),
};
for (var i = 0; i < threadCount; i++)
{
var subBuilder = ProcessBuilder.Fluent;
var subMutatorBuilder = subBuilder.ReadFrom(splitter);
mutatorBuilder.Invoke(i, subMutatorBuilder);
var subProcess = subBuilder.Result;
merger.ProcessList.Add(subProcess);
}
builder.ProcessBuilder.Result = merger;
return(builder);
}
public static IFluentProcessMutatorBuilder RemoveAllRow(this IFluentProcessMutatorBuilder builder)
{
return(builder.AddMutator(new RemoveRowMutator(builder.ProcessBuilder.Result.Context)
{
Name = nameof(RemoveAllRow),
}));
}
public static IFluentProcessMutatorBuilder CreateTaggedVersions(this IFluentProcessMutatorBuilder builder, params object[] tags)
{
return(builder.AddMutator(new MultiplyByTagsMutator(builder.ProcessBuilder.Result.Context)
{
Tags = tags,
}));
}
public static IFluentProcessMutatorBuilder TrimString(this IFluentProcessMutatorBuilder builder, params string[] columns)
{
return(builder.AddMutator(new TrimStringMutator(builder.ProcessBuilder.Result.Context)
{
Columns = columns,
}));
}
public static IFluentProcessMutatorBuilder Explode(this IFluentProcessMutatorBuilder builder, string name, ExplodeDelegate rowCreator)
{
return(builder.AddMutator(new ExplodeMutator(builder.ProcessBuilder.Result.Context)
{
Name = name,
RowCreator = rowCreator,
}));
}
/// <summary>
/// Keeps only the first row of each key, and discard all subsequent rows with existing keys.
/// <para>- input can be unordered</para>
/// <para>- if a more refined logic is required to decide which row should be kept of rows with same key then <see cref="ReduceGroupToSingleRowMutatorFluent.ReduceGroupToSingleRow(IFluentProcessMutatorBuilder, ReduceGroupToSingleRowMutator)"/> or <see cref="SortedReduceGroupToSingleRowMutatorFluent.ReduceGroupToSingleRowOrdered(IFluentProcessMutatorBuilder, SortedReduceGroupToSingleRowMutator)"/></para> can be used instead.
/// <para>- all keys are stored in memory</para>
/// </summary>
public static IFluentProcessMutatorBuilder RemoveDuplicateRows(this IFluentProcessMutatorBuilder builder, string name, Func <IReadOnlyRow, string> keyGenerator)
{
return(builder.AddMutator(new RemoveDuplicateRowsMutator(builder.ProcessBuilder.Result.Context)
{
Name = name,
KeyGenerator = keyGenerator,
}));
}
public static IFluentProcessMutatorBuilder SortInMemory(this IFluentProcessMutatorBuilder builder, string name, Func <IEnumerable <IRow>, IEnumerable <IRow> > sorter)
{
return(builder.AddMutator(new MemorySortMutator(builder.ProcessBuilder.Result.Context)
{
Name = name,
Sorter = sorter,
}));
}
public static IFluentProcessMutatorBuilder RemoveRow(this IFluentProcessMutatorBuilder builder, string name, RowTestDelegate rowTestDelegate)
{
return(builder.AddMutator(new RemoveRowMutator(builder.ProcessBuilder.Result.Context)
{
Name = name,
RowFilter = rowTestDelegate,
}));
}
public static IProducer BuildToInMemoryRowCache(this IFluentProcessMutatorBuilder builder, string name = null)
{
return(new InMemoryRowCache(builder.ProcessBuilder.Result.Context)
{
Name = name,
InputProcess = builder.Build(),
});
}
public static IFluentProcessMutatorBuilder CustomCode(this IFluentProcessMutatorBuilder builder, string name, Action <IRow> action)
{
return(builder.AddMutator(new CustomMutator(builder.ProcessBuilder.Result.Context)
{
Action = row =>
{
action.Invoke(row);
return true;
}
}));
}
public static IFluentProcessMutatorBuilder CreateTaggedVersions(this IFluentProcessMutatorBuilder builder, MultiplyByTagsMutator mutator)
{
return(builder.AddMutator(mutator));
}
public static IFluentProcessMutatorBuilder ThrowExceptionOnRowError(this IFluentProcessMutatorBuilder builder, ThrowExceptionOnRowErrorMutator mutator)
{
return(builder.AddMutators(mutator));
}
public static IFluentProcessMutatorBuilder SetTag(this IFluentProcessMutatorBuilder builder, SetTagMutator mutator)
{
return(builder.AddMutator(mutator));
}
public static IFluentProcessMutatorBuilder RemoveColumn(this IFluentProcessMutatorBuilder builder, RemoveColumnMutator mutator)
{
return(builder.AddMutators(mutator));
}
public static IFluentProcessMutatorBuilder ReplaceEmptyStringWithNull(this IFluentProcessMutatorBuilder builder, ReplaceEmptyStringWithNullMutator mutator)
{
return(builder.AddMutator(mutator));
}
public static IFluentProcessMutatorBuilder ReplaceNullWithValue(this IFluentProcessMutatorBuilder builder, TrimStringMutator mutator)
{
return(builder.AddMutators(mutator));
}
public static IFluentProcessMutatorBuilder MergeStringColumns(this IFluentProcessMutatorBuilder builder, MergeStringColumnsMutator mutator)
{
return(builder.AddMutator(mutator));
}
public static IFluentProcessMutatorBuilder RemoveRow(this IFluentProcessMutatorBuilder builder, RemoveRowWithErrorMutator mutator)
{
return(builder.AddMutator(mutator));
}
/// <summary>
/// Keeps only the first row of each key, and discard all subsequent rows with existing keys.
/// <para>- input can be unordered</para>
/// <para>- if a more refined logic is required to decide which row should be kept of rows with same key then <see cref="ReduceGroupToSingleRowMutatorFluent.ReduceGroupToSingleRow(IFluentProcessMutatorBuilder, ReduceGroupToSingleRowMutator)"/> or <see cref="SortedReduceGroupToSingleRowMutatorFluent.ReduceGroupToSingleRowOrdered(IFluentProcessMutatorBuilder, SortedReduceGroupToSingleRowMutator)"/></para> can be used instead.
/// <para>- all keys are stored in memory</para>
/// </summary>
public static IFluentProcessMutatorBuilder RemoveDuplicateRows(this IFluentProcessMutatorBuilder builder, RemoveDuplicateRowsMutator mutator)
{
return(builder.AddMutators(mutator));
}
public static IFluentProcessMutatorBuilder TrimString(this IFluentProcessMutatorBuilder builder, TrimStringMutator mutator)
{
return(builder.AddMutator(mutator));
}
public static IFluentProcessMutatorBuilder SortInMemory(this IFluentProcessMutatorBuilder builder, MemorySortMutator mutator)
{
return(builder.AddMutator(mutator));
}
/// <summary>
/// <para>- input can be unordered</para>
/// <para>- returns all aggregates at once when everything is processed (blocks execution)</para>
/// <para>- memory footprint is high because all rows in all groups are collected before aggregation</para>
/// <para>- if the input is ordered then <see cref="SortedMemoryAggregationMutatorFluent.AggregateOrdered(IFluentProcessMutatorBuilder, SortedMemoryAggregationMutator)"/> should be used for much lower memory footprint and stream-like behavior</para>
/// <para>- if the input is unordered but only basic operations are used then <see cref="ContinuousAggregationMutatorFluent.AggregateContinuously(IFluentProcessMutatorBuilder, ContinuousAggregationMutator)"/> should be used</para>
/// </summary>
public static IFluentProcessMutatorBuilder Aggregate(this IFluentProcessMutatorBuilder builder, MemoryAggregationMutator mutator)
{
return(builder.AddMutators(mutator));
}
public static IFluentProcessMutatorBuilder ThrowExceptionOnRowError(this IFluentProcessMutatorBuilder builder)
{
return(builder.AddMutator(new ThrowExceptionOnRowErrorMutator(builder.ProcessBuilder.Result.Context)));
}
/// <summary>
/// <para>- input must be ordered by group key</para>
/// <para>- returns each aggregate right after a group is processed (stream-like behavior like most mutators)</para>
/// <para>- if there is an ordering mismatch in the input then later appearances of a previously processed key will create new aggregated group(s)</para>
/// <para>- memory footprint is very low because only rows of one group are collected before aggregation is executed on them</para>
/// </summary>
public static IFluentProcessMutatorBuilder AggregateOrdered(this IFluentProcessMutatorBuilder builder, SortedMemoryAggregationMutator mutator)
{
return(builder.AddMutator(mutator));
}
public static IFluentProcessMutatorBuilder CustomCode(this IFluentProcessMutatorBuilder builder, CustomMutator mutator)
{
return(builder.AddMutators(mutator));
}
/// <summary>
/// Throw an exception if a subsequent occurence of a row key is found.
/// <para>- input can be unordered</para>
/// <para>- all keys are stored in memory</para>
/// </summary>
public static IFluentProcessMutatorBuilder ThrowExceptionOnDuplicateKey(this IFluentProcessMutatorBuilder builder, ThrowExceptionOnDuplicateKeyMutator mutator)
{
return(builder.AddMutator(mutator));
}
public static IFluentProcessMutatorBuilder Explode(this IFluentProcessMutatorBuilder builder, ExplodeMutator mutator)
{
return(builder.AddMutators(mutator));
}
