private static EpPlusExcelReader GetReader(IEtlContext context, string fileName, string sheetName = null, int sheetIndex = -1, bool automaticallyTrimAllStringValues = true)
{
return(new EpPlusExcelReader(context)
{
FileName = fileName,
Columns = new()
{
internal SequentialMergerBuilder(IEtlContext context, string name)
{
Merger = new SequentialMerger(context)
{
Name = name,
SequenceList = new List <ISequence>(),
};
}
internal SequentialMergerBuilder(IEtlContext context, string name)
{
Merger = new SequentialMerger(context)
{
Name = name,
ProcessList = new List <IProducer>(),
};
}
private static EpPlusExcelReader GetReader(IEtlContext context, IStreamProvider streamProvider, string sheetName = null, int sheetIndex = -1, bool automaticallyTrimAllStringValues = true)
{
return(new EpPlusExcelReader(context)
{
StreamProvider = streamProvider,
SheetName = sheetName,
SheetIndex = sheetIndex,
Columns = new()
{
private static EpPlusExcelSheetListReader GetReader(IEtlContext context, string fileName)
{
return(new EpPlusExcelSheetListReader(context)
{
StreamProvider = new LocalFileStreamProvider()
{
FileName = fileName,
},
AddRowIndexToColumn = "idx",
});
}
public EtlTransactionScope(IEtlContext context, IProcess process, TransactionScopeKind kind, TimeSpan scopeTimeout, LogSeverity logSeverity)
{
Context = context;
Process = process;
Kind = kind;
LogSeverity = logSeverity;
Timeout = scopeTimeout;
if (Kind == TransactionScopeKind.None)
{
return;
}
var previousId = Transaction.Current?.ToIdentifierString();
if (Kind == TransactionScopeKind.Suppress && previousId == null)
{
return;
}
Scope = new TransactionScope((TransactionScopeOption)Kind, scopeTimeout);
var newId = Transaction.Current?.ToIdentifierString();
var iocUid = 0;
switch (kind)
{
case TransactionScopeKind.RequiresNew:
iocUid = Context.RegisterIoCommandStart(Process, IoCommandKind.dbTransaction, null, null, "new transaction started", newId, null,
"new transaction started");
break;
case TransactionScopeKind.Required:
iocUid = previousId == null || newId != previousId
? Context.RegisterIoCommandStart(Process, IoCommandKind.dbTransaction, null, null, "new transaction started", newId, null,
"new transaction started")
: Context.RegisterIoCommandStart(Process, IoCommandKind.dbTransaction, null, null, "new transaction started and merged with previous", newId, () => new[] { new KeyValuePair <string, object>("previous transaction", previousId) },
"new transaction started and merged with previous");
break;
case TransactionScopeKind.Suppress:
iocUid = Context.RegisterIoCommandStart(Process, IoCommandKind.dbTransaction, null, null, "transaction suppressed", previousId, null,
"existing transaction suppressed");
break;
}
Context.RegisterIoCommandSuccess(Process, IoCommandKind.dbTransaction, iocUid, null);
}
public static IProducer PersonalAssetsPivot(IEtlContext context)
{
return(new RowCreator(context)
{
Columns = PersonalAssetsPivotColumns,
InputRows = new List <object[]>()
{
// "id", "personName", "cars", "houses", "kids"
new object[] { 1, "A", 1, 1, 2 },
new object[] { null, "C", 2, 1, 3 },
new object[] { 3, "D", null, 1, 3 },
new object[] { 4, "E", "6", 1, 3 },
},
});
}
public static IProducer RoleHierarchy(IEtlContext context)
{
return(new RowCreator(context)
{
Columns = RoleHierarchyColumns,
InputRows = new List <object[]>()
{
// "id", "code", "level1", "level2", "level3"
new object[] { 0, "A", "AAA" },
new object[] { 1, "B", null, "BBB" },
new object[] { 2, "C", null, null, "CCC" },
new object[] { 3, "D", null, null, "DDD" },
new object[] { 4, "E", null, "EEE" },
new object[] { 5, "F", null, "FFF" },
},
});
}
public static IProducer PersonEyeColor(IEtlContext context)
{
return(new RowCreator(context)
{
Columns = PersonEyeColorColumns,
InputRows = new List <object[]>()
{
// "id", "personId", "color"
new object[] { 0, 0, "yellow" },
new object[] { 1, 0, "red" },
new object[] { 2, 0, "green" },
new object[] { 3, 1, "blue" },
new object[] { 4, 1, "yellow" },
new object[] { 5, 2, "black" },
new object[] { 6, 100, "fake" },
},
});
}
private static void LogTaskSummary(IEtlContext context, TaskExectionResult result, int longestTaskName)
{
var spacing1 = "".PadRight(longestTaskName - result.TaskName.Length);
var spacing1WithoutName = "".PadRight(longestTaskName);
if (result.Exceptions.Count == 0)
{
context.Log(LogSeverity.Information, null, "{Task}{spacing1} run-time is {Elapsed}, result is {Result}, CPU time: {CpuTime}, total allocations: {AllocatedMemory}, allocation difference: {MemoryDifference}",
result.TaskName, spacing1, result.Statistics.RunTime, "success", result.Statistics.CpuTime, result.Statistics.TotalAllocations, result.Statistics.AllocationDifference);
}
else
{
context.Log(LogSeverity.Information, null, "{Task}{spacing1} run-time is {Elapsed}, result is {Result}, CPU time: {CpuTime}, total allocations: {AllocatedMemory}, allocation difference: {MemoryDifference}",
result.TaskName, spacing1, result.Statistics.RunTime, "failed", result.Statistics.CpuTime, result.Statistics.TotalAllocations, result.Statistics.AllocationDifference);
}
if (result.IoCommandCounters.Count > 0)
{
var maxKeyLength = result.IoCommandCounters.Max(x => x.Key.ToString().Length);
var maxInvocationLength = result.IoCommandCounters.Max(x => x.Value.InvocationCount.ToString(SerilogSink.ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length);
foreach (var kvp in result.IoCommandCounters.OrderBy(kvp => kvp.Key.ToString()))
{
if (kvp.Value.AffectedDataCount != null)
{
context.Log(LogSeverity.Information, null, "{spacing1} {Kind}{spacing2} {InvocationCount}{spacing3} {AffectedDataCount}", spacing1WithoutName,
kvp.Key.ToString(),
"".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '),
kvp.Value.InvocationCount,
"".PadRight(maxInvocationLength - kvp.Value.InvocationCount.ToString(SerilogSink.ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length, ' '),
kvp.Value.AffectedDataCount);
}
else
{
context.Log(LogSeverity.Information, null, "{spacing1} {Kind}{spacing2} {InvocationCount}", spacing1WithoutName,
kvp.Key.ToString(),
"".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '),
kvp.Value.InvocationCount);
}
}
}
}
private static void LogTaskCounters(IEtlContext context, IEtlTask task)
{
if (task.IoCommandCounters.Count == 0)
{
return;
}
const string kind = "kind";
const string invocation = "invoc.";
const string affected = "affected";
var maxKeyLength = Math.Max(kind.Length, task.IoCommandCounters.Max(x => x.Key.ToString().Length));
var maxInvocationLength = Math.Max(invocation.Length, task.IoCommandCounters.Max(x => x.Value.InvocationCount.ToString(SerilogSink.ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length));
context.Log(LogSeverity.Debug, task, "{Kind}{spacing1} {InvocationCount}{spacing2} {AffectedDataCount}", kind,
"".PadRight(maxKeyLength - kind.Length, ' '),
invocation,
"".PadRight(maxInvocationLength - invocation.Length, ' '),
affected);
foreach (var kvp in task.IoCommandCounters.OrderBy(kvp => kvp.Key.ToString()))
{
if (kvp.Value.AffectedDataCount != null)
{
context.Log(LogSeverity.Debug, task, "{Kind}{spacing1} {InvocationCount}{spacing2} {AffectedDataCount}", kvp.Key.ToString(),
"".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '),
kvp.Value.InvocationCount,
"".PadRight(maxInvocationLength - kvp.Value.InvocationCount.ToString(SerilogSink.ValueFormatter.DefaultIntegerFormat, CultureInfo.InvariantCulture).Length, ' '),
kvp.Value.AffectedDataCount);
}
else
{
context.Log(LogSeverity.Debug, task, "{Kind}{spacing1} {InvocationCount}", kvp.Key.ToString(),
"".PadRight(maxKeyLength - kvp.Key.ToString().Length, ' '), kvp.Value.InvocationCount);
}
}
}
public SortedMemoryAggregationMutator(IEtlContext context)
: base(context)
{
}
public CustomJob(IEtlContext context)
: base(context)
{
}
protected AbstractSimpleChangeMutator(IEtlContext context)
: base(context)
{
}
public ParallelMerger(IEtlContext context)
: base(context)
{
}
protected AbstractMutator(IEtlContext context)
: base(context)
{
}
public CustomAction(IEtlContext context)
: base(context)
{
}
public RowListener(IEtlContext context)
: base(context)
{
}
protected AbstractJob(IEtlContext context)
: base(context)
{
}
public BasicScope(IEtlContext context, string topic, string name = null)
: base(context)
{
}
public InMemoryRowCache(IEtlContext context)
: base(context)
{
}
public DeleteFile(IEtlContext context)
: base(context)
{
}
