protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
var groups = new Dictionary <string, List <IReadOnlySlimRow> >();
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var rowCount = 0;
var ignoredRowCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
var apply = false;
try
{
apply = If?.Invoke(row) != false;
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
rowCount++;
var key = KeyGenerator.Invoke(row);
if (!groups.TryGetValue(key, out var list))
{
list = new List <IReadOnlySlimRow>();
groups.Add(key, list);
}
list.Add(row);
}
Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows and created {GroupCount} groups in {Elapsed}",
rowCount, groups.Count, InvocationInfo.LastInvocationStarted.Elapsed);
var aggregateCount = 0;
var aggregates = new List <SlimRow>();
foreach (var group in groups.Values)
{
if (Context.CancellationTokenSource.IsCancellationRequested)
{
break;
}
try
{
Operation.TransformGroup(group, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, group, ex);
Context.AddException(this, exception);
break;
}
foreach (var row in group)
{
Context.SetRowOwner(row as IRow, null);
}
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
group.Clear();
aggregates.Clear();
}
groups.Clear();
netTimeStopwatch.Stop();
Context.Log(LogSeverity.Debug, this, "created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}",
aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount);
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
var groupRows = new List <IReadOnlySlimRow>();
string lastKey = null;
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var success = true;
var rowCount = 0;
var ignoredRowCount = 0;
var groupCount = 0;
var aggregateCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
var apply = false;
if (RowFilter != null)
{
try
{
apply = RowFilter.Invoke(row);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
if (RowTagFilter != null)
{
try
{
apply = RowTagFilter.Invoke(row.Tag);
}
catch (Exception ex)
{
Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
break;
}
if (!apply)
{
ignoredRowCount++;
netTimeStopwatch.Stop();
yield return(row);
netTimeStopwatch.Start();
continue;
}
}
rowCount++;
var key = KeyGenerator.Invoke(row);
if (key != lastKey)
{
lastKey = key;
if (groupRows.Count > 0)
{
var aggregates = new List <SlimRow>();
groupCount++;
try
{
Operation.TransformGroup(groupRows, () =>
{
var aggregate = new SlimRow
{
Tag = groupRows[0].Tag
};
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate[column.Key] = groupRows[0][column.Value ?? column.Key];
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
Context.AddException(this, exception);
success = false;
break;
}
foreach (var groupRow in groupRows)
{
Context.SetRowOwner(groupRow as IRow, null);
}
groupRows.Clear();
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
groupRows.Add(row);
}
if (success && groupRows.Count > 0)
{
var aggregates = new List <SlimRow>();
groupCount++;
try
{
Operation.TransformGroup(groupRows, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var col in FixColumns)
{
aggregate[col.Key] = groupRows[0][col.Value ?? col.Key];
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
Context.AddException(this, exception);
success = false;
}
foreach (var groupRow in groupRows)
{
Context.SetRowOwner(groupRow as IRow, null);
}
groupRows.Clear();
if (success)
{
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
netTimeStopwatch.Stop();
Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows, created {GroupCount} groups and created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}",
rowCount, groupCount, aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount);
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}
protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
{
var group = new List <IReadOnlySlimRow>();
string lastKey = null;
netTimeStopwatch.Stop();
var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();
netTimeStopwatch.Start();
var success = true;
var rowCount = 0;
var aggregateCount = 0;
while (!Context.CancellationTokenSource.IsCancellationRequested)
{
netTimeStopwatch.Stop();
var finished = !enumerator.MoveNext();
netTimeStopwatch.Start();
if (finished)
{
break;
}
var row = enumerator.Current;
rowCount++;
var key = KeyGenerator.Invoke(row);
if (key != lastKey)
{
lastKey = key;
if (group.Count > 0)
{
var aggregates = new List <SlimRow>();
try
{
Operation.TransformGroup(group, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, group, ex);
Context.AddException(this, exception);
success = false;
break;
}
foreach (var groupRow in group)
{
Context.SetRowOwner(groupRow as IRow, null);
}
group.Clear();
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
group.Add(row);
}
if (success && group.Count > 0)
{
var aggregates = new List <SlimRow>();
try
{
Operation.TransformGroup(group, () =>
{
var aggregate = new SlimRow();
if (FixColumns != null)
{
foreach (var column in FixColumns)
{
aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
}
}
aggregates.Add(aggregate);
return(aggregate);
});
}
catch (Exception ex)
{
var exception = new MemoryAggregationException(this, Operation, group, ex);
Context.AddException(this, exception);
success = false;
}
foreach (var groupRow in group)
{
Context.SetRowOwner(groupRow as IRow, null);
}
group.Clear();
if (success)
{
foreach (var aggregate in aggregates)
{
aggregateCount++;
var aggregateRow = Context.CreateRow(this, aggregate);
netTimeStopwatch.Stop();
yield return(aggregateRow);
netTimeStopwatch.Start();
}
}
}
Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows and created {GroupCount} groups in {Elapsed}",
rowCount, group.Count, InvocationInfo.LastInvocationStarted.Elapsed);
netTimeStopwatch.Stop();
Context.Log(LogSeverity.Debug, this, "created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}",
aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed);
Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
}