protected override void ExecuteImpl()
{
using (var clt = new WebClient())
{
var iocUid = 0;
try
{
using (Context.CancellationTokenSource.Token.Register(clt.CancelAsync))
{
iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.httpGet, Url, null, null, null, null,
"downloading file from {Url} to {FileName}",
Url, PathHelpers.GetFriendlyPathName(FileName));
clt.DownloadFile(Url, FileName);
Context.RegisterIoCommandSuccess(this, IoCommandKind.httpGet, iocUid, Convert.ToInt32(new FileInfo(FileName).Length));
}
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.httpGet, iocUid, null, ex);
var exception = new ProcessExecutionException(this, "file download failed", ex);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "file download failed, url: {0}, file name: {1}, message: {2}",
Url, FileName, ex.Message));
exception.Data.Add("Url", Url);
exception.Data.Add("FileName", FileName);
throw exception;
}
}
}
protected override void ExecuteImpl()
{
if (!File.Exists(FileName))
{
Context.Log(LogSeverity.Debug, this, "can't delete file because it doesn't exist '{FileName}'", PathHelpers.GetFriendlyPathName(FileName));
return;
}
Context.Log(LogSeverity.Information, this, "deleting file '{FileName}'", PathHelpers.GetFriendlyPathName(FileName));
try
{
File.Delete(FileName);
Context.Log(LogSeverity.Debug, this, "successfully deleted file '{FileName}' in {Elapsed}", PathHelpers.GetFriendlyPathName(FileName),
InvocationInfo.LastInvocationStarted.Elapsed);
}
catch (Exception ex)
{
var exception = new ProcessExecutionException(this, "file deletion failed", ex);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "file deletion failed, file name: {0}, message: {1}",
FileName, ex.Message));
exception.Data.Add("FileName", FileName);
throw exception;
}
}
public void Execute(IProcess caller = null)
{
Context.RegisterProcessInvocationStart(this, caller);
var netTimeStopwatch = Stopwatch.StartNew();
try
{
ValidateImpl();
if (Context.CancellationTokenSource.IsCancellationRequested)
{
return;
}
ExecuteImpl();
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, ex));
}
finally
{
netTimeStopwatch.Stop();
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
}
public void Append(ICountableLookup lookup, IProcess caller, IReadOnlySlimRow[] filterRows)
{
var process = ProcessCreator.Invoke(filterRows);
var rows = process.Evaluate(caller).TakeRowsAndReleaseOwnership();
var rowCount = 0;
foreach (var row in rows)
{
rowCount++;
string key = null;
try
{
key = KeyGenerator(row);
}
catch (EtlException) { throw; }
catch (Exception)
{
var exception = new ProcessExecutionException(caller, row, nameof(RowLookupBuilder) + " failed");
throw exception;
}
if (string.IsNullOrEmpty(key))
{
continue;
}
lookup.AddRow(key, row);
}
caller?.Context.Log(LogSeverity.Debug, caller, "fetched {RowCount} rows, lookup size is {LookupSize}",
rowCount, lookup.Count);
}
public void AddException(IProcess process, Exception ex)
{
if (ex is OperationCanceledException)
{
return;
}
if (!(ex is EtlException))
{
ex = new ProcessExecutionException(process, ex);
}
Result.Exceptions.Add(ex);
lock (_exceptions)
{
if (_exceptions.Contains(ex))
{
CancellationTokenSource.Cancel();
return;
}
_exceptions.Add(ex);
}
foreach (var listener in Listeners)
{
listener.OnException(process, ex);
}
CancellationTokenSource.Cancel();
}
public void Append(ICountableLookup lookup, IProcess caller)
{
var allRows = Process.Evaluate(caller).TakeRowsAndReleaseOwnership();
var rowCount = 0;
foreach (var row in allRows)
{
rowCount++;
string key = null;
try
{
key = KeyGenerator(row);
}
catch (EtlException) { throw; }
catch (Exception)
{
var exception = new ProcessExecutionException(caller, row, nameof(RowLookupBuilder) + " failed");
throw exception;
}
lookup.AddRow(key, row);
}
caller?.Context.Log(LogSeverity.Debug, caller, "fetched {RowCount} rows, lookup size is {LookupSize}",
rowCount, lookup.Count);
}
public void Execute(IProcess caller = null)
{
Context.RegisterProcessInvocationStart(this, caller);
if (caller != null)
{
Context.Log(LogSeverity.Information, this, "process started by {Process}", caller.Name);
}
else
{
Context.Log(LogSeverity.Information, this, "process started");
}
LogPublicSettableProperties(LogSeverity.Verbose);
var netTimeStopwatch = Stopwatch.StartNew();
try
{
ValidateImpl();
if (Context.CancellationTokenSource.IsCancellationRequested)
{
return;
}
ExecuteImpl();
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, ex));
}
finally
{
netTimeStopwatch.Stop();
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
}
protected override IEnumerable <IRow> Produce()
{
var iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.fileRead, FileName, null, null, null, null,
"reading from {FileName}",
PathHelpers.GetFriendlyPathName(FileName));
if (!File.Exists(FileName))
{
var exception = new ProcessExecutionException(this, "input file doesn't exist");
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "input file doesn't exist: {0}",
FileName));
exception.Data.Add("FileName", FileName);
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, 0, exception);
throw exception;
}
var columnConfig = ColumnConfiguration?.ToDictionary(x => x.SourceColumn.ToUpperInvariant(), StringComparer.OrdinalIgnoreCase);
var resultCount = 0;
Stream stream;
StreamReader reader;
try
{
stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.Read);
reader = new StreamReader(stream);
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, null, ex);
var exception = new EtlException(this, "error while opening file", ex);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while opening file: {0}, message: {1}", FileName, ex.Message));
exception.Data.Add("FileName", FileName);
throw exception;
}
var firstRow = true;
var initialValues = new List <KeyValuePair <string, object> >();
var partList = new List <string>(100);
var builder = new StringBuilder(2000);
// capture for performance
var columnNames = ColumnNames;
var delimiter = Delimiter;
var treatEmptyStringAsNull = TreatEmptyStringAsNull;
var removeSurroundingDoubleQuotes = RemoveSurroundingDoubleQuotes;
var throwOnMissingDoubleQuoteClose = ThrowOnMissingDoubleQuoteClose;
var ignoreColumns = IgnoreColumns?.ToHashSet();
try
{
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
string line;
try
{
line = reader.ReadLine();
if (line == null)
{
break;
}
if (string.IsNullOrEmpty(line))
{
continue;
}
}
catch (Exception ex)
{
Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, resultCount, ex);
var exception = new EtlException(this, "error while reading data from file", ex);
exception.Data.Add("FileName", FileName);
exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while reading data from file: {0}, message: {1}", FileName, ex.Message));
throw exception;
}
if (line.EndsWith(delimiter))
{
line = line[0..^ 1];
protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
var groupRows = new List <IReadOnlySlimRow>();
string lastKey = null;
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var success = true;
var rowCount = 0;
var ignoredRowCount = 0;
var groupCount = 0;
var aggregateCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
var apply = false;
if (RowFilter != null)
{
try
{
apply = RowFilter.Invoke(row);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
if (RowTagFilter != null)
{
try
{
apply = RowTagFilter.Invoke(row.Tag);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
rowCount++;
var key = KeyGenerator.Invoke(row);
if (key != lastKey)
{
lastKey = key;
if (groupRows.Count > 0)
{
var aggregates = new List <SlimRow>();
groupCount++;
try
{
Operation.TransformGroup(groupRows, () =>
{
var aggregate = new SlimRow
{
Tag = groupRows[0].Tag
};
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate[column.Key] = groupRows[0][column.Value ?? column.Key];
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
Context.AddException(this, exception);
success = false;
break;
}
foreach (var groupRow in groupRows)
{
Context.SetRowOwner(groupRow as IRow, null);
}
groupRows.Clear();
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
groupRows.Add(row);
}
if (success && groupRows.Count > 0)
{
var aggregates = new List <SlimRow>();
groupCount++;
try
{
Operation.TransformGroup(groupRows, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var col in FixColumns)
{
aggregate[col.Key] = groupRows[0][col.Value ?? col.Key];
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
Context.AddException(this, exception);
success = false;
}
foreach (var groupRow in groupRows)
{
Context.SetRowOwner(groupRow as IRow, null);
}
groupRows.Clear();
if (success)
{
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
netTimeStopwatch.Stop();
Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows, created {GroupCount} groups and created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}",
rowCount, groupCount, aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount);
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
protected sealed override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
try
{
StartMutator();
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, ex));
netTimeStopwatch.Stop();
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
yield break;
}
var mutatedRows = new List <IRow>();
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var mutatedRowCount = 0;
var ignoredRowCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
var apply = false;
if (RowFilter != null)
{
try
{
apply = RowFilter.Invoke(row);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
if (RowTagFilter != null)
{
try
{
apply = RowTagFilter.Invoke(row.Tag);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
mutatedRowCount++;
var kept = false;
try
{
foreach (var mutatedRow in MutateRow(row))
{
if (mutatedRow == row)
{
kept = true;
}
if (mutatedRow.CurrentProcess != this)
{
Context.AddException(this, new ProcessExecutionException(this, mutatedRow, "mutator returned a row without proper ownership"));
break;
}
mutatedRows.Add(mutatedRow);
}
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!kept)
{
Context.SetRowOwner(row, null);
}
netTimeStopwatch.Stop();
foreach (var mutatedRow in mutatedRows)
{
yield return(mutatedRow);
}
netTimeStopwatch.Start();
mutatedRows.Clear();
}
try
{
CloseMutator();
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, ex));
netTimeStopwatch.Stop();
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
yield break;
}
netTimeStopwatch.Stop();
if (mutatedRowCount + ignoredRowCount > 0)
{
Context.Log(LogSeverity.Debug, this, "mutated {MutatedRowCount} of {TotalRowCount} rows in {Elapsed}/{ElapsedWallClock}",
mutatedRowCount, mutatedRowCount + ignoredRowCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed);
}
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
var groups = new Dictionary <string, List <IReadOnlySlimRow> >();
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var rowCount = 0;
var ignoredRowCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
var apply = false;
try
{
apply = If?.Invoke(row) != false;
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
rowCount++;
var key = KeyGenerator.Invoke(row);
if (!groups.TryGetValue(key, out var list))
{
list = new List <IReadOnlySlimRow>();
groups.Add(key, list);
}
list.Add(row);
}
Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows and created {GroupCount} groups in {Elapsed}",
rowCount, groups.Count, InvocationInfo.LastInvocationStarted.Elapsed);
var aggregateCount = 0;
var aggregates = new List <SlimRow>();
foreach (var group in groups.Values)
{
if (Context.CancellationTokenSource.IsCancellationRequested)
{
break;
}
try
{
Operation.TransformGroup(group, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, group, ex);
Context.AddException(this, exception);
break;
}
foreach (var row in group)
{
Context.SetRowOwner(row as IRow, null);
}
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
group.Clear();
aggregates.Clear();
}
groups.Clear();
netTimeStopwatch.Stop();
Context.Log(LogSeverity.Debug, this, "created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}",
aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount);
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}