protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch) { var groupRows = new List <IReadOnlySlimRow>(); string lastKey = null; netTimeStopwatch.Stop(); var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator(); netTimeStopwatch.Start(); var success = true; var rowCount = 0; var ignoredRowCount = 0; var groupCount = 0; var aggregateCount = 0; while (!Context.CancellationTokenSource.IsCancellationRequested) { netTimeStopwatch.Stop(); var finished = !enumerator.MoveNext(); netTimeStopwatch.Start(); if (finished) { break; } var row = enumerator.Current; var apply = false; if (RowFilter != null) { try { apply = RowFilter.Invoke(row); } catch (Exception ex) { Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex)); break; } if (!apply) { ignoredRowCount++; netTimeStopwatch.Stop(); yield return(row); netTimeStopwatch.Start(); continue; } } if (RowTagFilter != null) { try { apply = RowTagFilter.Invoke(row.Tag); } catch (Exception ex) { Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex)); break; } if (!apply) { ignoredRowCount++; netTimeStopwatch.Stop(); yield return(row); netTimeStopwatch.Start(); continue; } } rowCount++; var key = KeyGenerator.Invoke(row); if (key != lastKey) { lastKey = key; if (groupRows.Count > 0) { var aggregates = new List <SlimRow>(); groupCount++; try { Operation.TransformGroup(groupRows, () => { var aggregate = new SlimRow { Tag = groupRows[0].Tag }; if (FixColumns != null) { foreach (var column in FixColumns) { aggregate[column.Key] = groupRows[0][column.Value ?? column.Key]; } } aggregates.Add(aggregate); return(aggregate); }); } catch (Exception ex) { var exception = new MemoryAggregationException(this, Operation, groupRows, ex); Context.AddException(this, exception); success = false; break; } foreach (var groupRow in groupRows) { Context.SetRowOwner(groupRow as IRow, null); } groupRows.Clear(); foreach (var aggregate in aggregates) { aggregateCount++; var aggregateRow = Context.CreateRow(this, aggregate); netTimeStopwatch.Stop(); yield return(aggregateRow); netTimeStopwatch.Start(); } } } groupRows.Add(row); } if (success && groupRows.Count > 0) { var aggregates = new List <SlimRow>(); groupCount++; try { Operation.TransformGroup(groupRows, () => { var aggregate = new SlimRow(); if (FixColumns != null) { foreach (var col in FixColumns) { aggregate[col.Key] = groupRows[0][col.Value ?? col.Key]; } } aggregates.Add(aggregate); return(aggregate); }); } catch (Exception ex) { var exception = new MemoryAggregationException(this, Operation, groupRows, ex); Context.AddException(this, exception); success = false; } foreach (var groupRow in groupRows) { Context.SetRowOwner(groupRow as IRow, null); } groupRows.Clear(); if (success) { foreach (var aggregate in aggregates) { aggregateCount++; var aggregateRow = Context.CreateRow(this, aggregate); netTimeStopwatch.Stop(); yield return(aggregateRow); netTimeStopwatch.Start(); } } } netTimeStopwatch.Stop(); Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows, created {GroupCount} groups and created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}", rowCount, groupCount, aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount); Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds); }
protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch) { var groups = new Dictionary <string, List <IReadOnlySlimRow> >(); var groupCount = 0; netTimeStopwatch.Stop(); var enumerator = Input.TakeRowsAndTransferOwnership(this).GetEnumerator(); netTimeStopwatch.Start(); var rowCount = 0; var ignoredRowCount = 0; while (!Context.IsTerminating) { netTimeStopwatch.Stop(); var finished = !enumerator.MoveNext(); if (finished) { break; } var row = enumerator.Current; netTimeStopwatch.Start(); if (row.Tag is HeartBeatTag) { netTimeStopwatch.Stop(); yield return(row); netTimeStopwatch.Start(); continue; } var apply = false; if (RowFilter != null) { try { apply = RowFilter.Invoke(row); } catch (Exception ex) { AddException(ex, row); break; } if (!apply) { ignoredRowCount++; netTimeStopwatch.Stop(); yield return(row); netTimeStopwatch.Start(); continue; } } if (RowTagFilter != null) { try { apply = RowTagFilter.Invoke(row.Tag); } catch (Exception ex) { AddException(ex, row); break; } if (!apply) { ignoredRowCount++; netTimeStopwatch.Stop(); yield return(row); netTimeStopwatch.Start(); continue; } } rowCount++; var key = KeyGenerator.Invoke(row); if (!groups.TryGetValue(key, out var list)) { list = new List <IReadOnlySlimRow>(); groups.Add(key, list); groupCount++; } list.Add(row); } netTimeStopwatch.Start(); var aggregateCount = 0; var aggregates = new List <SlimRow>(); foreach (var groupRows in groups.Values) { if (Context.IsTerminating) { break; } try { Operation.TransformGroup(groupRows, () => { var aggregate = new SlimRow { Tag = groupRows[0].Tag, }; if (FixColumns != null) { foreach (var column in FixColumns) { aggregate[column.Key] = groupRows[0][column.Value ?? column.Key]; } } aggregates.Add(aggregate); return(aggregate); }); } catch (Exception ex) { var exception = new MemoryAggregationException(this, Operation, groupRows, ex); AddException(exception); break; } foreach (var row in groupRows) { Context.SetRowOwner(row as IRow, null); } foreach (var aggregate in aggregates) { aggregateCount++; var aggregateRow = Context.CreateRow(this, aggregate); netTimeStopwatch.Stop(); yield return(aggregateRow); netTimeStopwatch.Start(); } groupRows.Clear(); aggregates.Clear(); } groups.Clear(); Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows, created {GroupCount} groups and created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}", rowCount, groupCount, aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount); }