Esempio n. 1
0
        protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
        {
            var groups = new Dictionary <string, List <IReadOnlySlimRow> >();

            netTimeStopwatch.Stop();
            var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();

            netTimeStopwatch.Start();

            var rowCount        = 0;
            var ignoredRowCount = 0;

            while (!Context.CancellationTokenSource.IsCancellationRequested)
            {
                netTimeStopwatch.Stop();
                var finished = !enumerator.MoveNext();
                netTimeStopwatch.Start();
                if (finished)
                {
                    break;
                }

                var row = enumerator.Current;

                var apply = false;
                try
                {
                    apply = If?.Invoke(row) != false;
                }
                catch (Exception ex)
                {
                    Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
                    break;
                }

                if (!apply)
                {
                    ignoredRowCount++;
                    netTimeStopwatch.Stop();
                    yield return(row);

                    netTimeStopwatch.Start();
                    continue;
                }

                rowCount++;
                var key = KeyGenerator.Invoke(row);
                if (!groups.TryGetValue(key, out var list))
                {
                    list = new List <IReadOnlySlimRow>();
                    groups.Add(key, list);
                }

                list.Add(row);
            }

            Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows and created {GroupCount} groups in {Elapsed}",
                        rowCount, groups.Count, InvocationInfo.LastInvocationStarted.Elapsed);

            var aggregateCount = 0;
            var aggregates     = new List <SlimRow>();

            foreach (var group in groups.Values)
            {
                if (Context.CancellationTokenSource.IsCancellationRequested)
                {
                    break;
                }

                try
                {
                    Operation.TransformGroup(group, () =>
                    {
                        var aggregate = new SlimRow();

                        if (FixColumns != null)
                        {
                            foreach (var column in FixColumns)
                            {
                                aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
                            }
                        }

                        aggregates.Add(aggregate);
                        return(aggregate);
                    });
                }
                catch (Exception ex)
                {
                    var exception = new MemoryAggregationException(this, Operation, group, ex);
                    Context.AddException(this, exception);
                    break;
                }

                foreach (var row in group)
                {
                    Context.SetRowOwner(row as IRow, null);
                }

                foreach (var aggregate in aggregates)
                {
                    aggregateCount++;
                    var aggregateRow = Context.CreateRow(this, aggregate);

                    netTimeStopwatch.Stop();
                    yield return(aggregateRow);

                    netTimeStopwatch.Start();
                }

                group.Clear();
                aggregates.Clear();
            }

            groups.Clear();

            netTimeStopwatch.Stop();
            Context.Log(LogSeverity.Debug, this, "created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}",
                        aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount);

            Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
        }
        protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
        {
            var    groupRows = new List <IReadOnlySlimRow>();
            string lastKey   = null;

            netTimeStopwatch.Stop();
            var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();

            netTimeStopwatch.Start();

            var success = true;

            var rowCount        = 0;
            var ignoredRowCount = 0;
            var groupCount      = 0;
            var aggregateCount  = 0;

            while (!Context.CancellationTokenSource.IsCancellationRequested)
            {
                netTimeStopwatch.Stop();
                var finished = !enumerator.MoveNext();
                netTimeStopwatch.Start();
                if (finished)
                {
                    break;
                }

                var row = enumerator.Current;

                var apply = false;
                if (RowFilter != null)
                {
                    try
                    {
                        apply = RowFilter.Invoke(row);
                    }
                    catch (Exception ex)
                    {
                        Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
                        break;
                    }

                    if (!apply)
                    {
                        ignoredRowCount++;
                        netTimeStopwatch.Stop();
                        yield return(row);

                        netTimeStopwatch.Start();
                        continue;
                    }
                }

                if (RowTagFilter != null)
                {
                    try
                    {
                        apply = RowTagFilter.Invoke(row.Tag);
                    }
                    catch (Exception ex)
                    {
                        Context.AddException(this, ProcessExecutionException.Wrap(this, row, ex));
                        break;
                    }

                    if (!apply)
                    {
                        ignoredRowCount++;
                        netTimeStopwatch.Stop();
                        yield return(row);

                        netTimeStopwatch.Start();
                        continue;
                    }
                }

                rowCount++;
                var key = KeyGenerator.Invoke(row);
                if (key != lastKey)
                {
                    lastKey = key;

                    if (groupRows.Count > 0)
                    {
                        var aggregates = new List <SlimRow>();
                        groupCount++;
                        try
                        {
                            Operation.TransformGroup(groupRows, () =>
                            {
                                var aggregate = new SlimRow
                                {
                                    Tag = groupRows[0].Tag
                                };

                                if (FixColumns != null)
                                {
                                    foreach (var column in FixColumns)
                                    {
                                        aggregate[column.Key] = groupRows[0][column.Value ?? column.Key];
                                    }
                                }

                                aggregates.Add(aggregate);
                                return(aggregate);
                            });
                        }
                        catch (Exception ex)
                        {
                            var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
                            Context.AddException(this, exception);
                            success = false;
                            break;
                        }

                        foreach (var groupRow in groupRows)
                        {
                            Context.SetRowOwner(groupRow as IRow, null);
                        }

                        groupRows.Clear();

                        foreach (var aggregate in aggregates)
                        {
                            aggregateCount++;
                            var aggregateRow = Context.CreateRow(this, aggregate);

                            netTimeStopwatch.Stop();
                            yield return(aggregateRow);

                            netTimeStopwatch.Start();
                        }
                    }
                }

                groupRows.Add(row);
            }

            if (success && groupRows.Count > 0)
            {
                var aggregates = new List <SlimRow>();
                groupCount++;
                try
                {
                    Operation.TransformGroup(groupRows, () =>
                    {
                        var aggregate = new SlimRow();

                        if (FixColumns != null)
                        {
                            foreach (var col in FixColumns)
                            {
                                aggregate[col.Key] = groupRows[0][col.Value ?? col.Key];
                            }
                        }

                        aggregates.Add(aggregate);
                        return(aggregate);
                    });
                }
                catch (Exception ex)
                {
                    var exception = new MemoryAggregationException(this, Operation, groupRows, ex);
                    Context.AddException(this, exception);
                    success = false;
                }

                foreach (var groupRow in groupRows)
                {
                    Context.SetRowOwner(groupRow as IRow, null);
                }

                groupRows.Clear();

                if (success)
                {
                    foreach (var aggregate in aggregates)
                    {
                        aggregateCount++;
                        var aggregateRow = Context.CreateRow(this, aggregate);

                        netTimeStopwatch.Stop();
                        yield return(aggregateRow);

                        netTimeStopwatch.Start();
                    }
                }
            }

            netTimeStopwatch.Stop();
            Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows, created {GroupCount} groups and created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}, ignored: {IgnoredRowCount}",
                        rowCount, groupCount, aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed, ignoredRowCount);

            Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
        }
        protected override IEnumerable <IRow> EvaluateImpl(Stopwatch netTimeStopwatch)
        {
            var    group   = new List <IReadOnlySlimRow>();
            string lastKey = null;

            netTimeStopwatch.Stop();
            var enumerator = InputProcess.Evaluate(this).TakeRowsAndTransferOwnership().GetEnumerator();

            netTimeStopwatch.Start();

            var success = true;

            var rowCount       = 0;
            var aggregateCount = 0;

            while (!Context.CancellationTokenSource.IsCancellationRequested)
            {
                netTimeStopwatch.Stop();
                var finished = !enumerator.MoveNext();
                netTimeStopwatch.Start();
                if (finished)
                {
                    break;
                }

                var row = enumerator.Current;
                rowCount++;
                var key = KeyGenerator.Invoke(row);
                if (key != lastKey)
                {
                    lastKey = key;

                    if (group.Count > 0)
                    {
                        var aggregates = new List <SlimRow>();

                        try
                        {
                            Operation.TransformGroup(group, () =>
                            {
                                var aggregate = new SlimRow();

                                if (FixColumns != null)
                                {
                                    foreach (var column in FixColumns)
                                    {
                                        aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
                                    }
                                }

                                aggregates.Add(aggregate);
                                return(aggregate);
                            });
                        }
                        catch (Exception ex)
                        {
                            var exception = new MemoryAggregationException(this, Operation, group, ex);
                            Context.AddException(this, exception);
                            success = false;
                            break;
                        }

                        foreach (var groupRow in group)
                        {
                            Context.SetRowOwner(groupRow as IRow, null);
                        }

                        group.Clear();

                        foreach (var aggregate in aggregates)
                        {
                            aggregateCount++;
                            var aggregateRow = Context.CreateRow(this, aggregate);

                            netTimeStopwatch.Stop();
                            yield return(aggregateRow);

                            netTimeStopwatch.Start();
                        }
                    }
                }

                group.Add(row);
            }

            if (success && group.Count > 0)
            {
                var aggregates = new List <SlimRow>();

                try
                {
                    Operation.TransformGroup(group, () =>
                    {
                        var aggregate = new SlimRow();

                        if (FixColumns != null)
                        {
                            foreach (var column in FixColumns)
                            {
                                aggregate.SetValue(column.ToColumn, group[0][column.FromColumn]);
                            }
                        }

                        aggregates.Add(aggregate);
                        return(aggregate);
                    });
                }
                catch (Exception ex)
                {
                    var exception = new MemoryAggregationException(this, Operation, group, ex);
                    Context.AddException(this, exception);
                    success = false;
                }

                foreach (var groupRow in group)
                {
                    Context.SetRowOwner(groupRow as IRow, null);
                }

                group.Clear();

                if (success)
                {
                    foreach (var aggregate in aggregates)
                    {
                        aggregateCount++;
                        var aggregateRow = Context.CreateRow(this, aggregate);

                        netTimeStopwatch.Stop();
                        yield return(aggregateRow);

                        netTimeStopwatch.Start();
                    }
                }
            }

            Context.Log(LogSeverity.Debug, this, "evaluated {RowCount} input rows and created {GroupCount} groups in {Elapsed}",
                        rowCount, group.Count, InvocationInfo.LastInvocationStarted.Elapsed);

            netTimeStopwatch.Stop();
            Context.Log(LogSeverity.Debug, this, "created {AggregateCount} aggregates in {Elapsed}/{ElapsedWallClock}",
                        aggregateCount, InvocationInfo.LastInvocationStarted.Elapsed, netTimeStopwatch.Elapsed);

            Context.RegisterProcessInvocationEnd(this, netTimeStopwatch.ElapsedMilliseconds);
        }