Пример #1
0
        public static Func <Task> ParallelForEach <T>(IQueueSource <T> source, ParallelWorker parallelWorker, Func <ForEachInfo <T>, Task> processAsync, Func <Task> onCloseAsync,
                                                      ExceptionCollector ec)
        {
            Func <Task> t = async delegate {
                IdleDetector idleDetector = new IdleDetector();
                try {
                    while (true)
                    {
                        var item = await source.Dequeue(ec.CancellationToken);

                        if (!item.HasValue)
                        {
                            break;
                        }
                        T itemValue = item.Value;

                        idleDetector.Enter();

                        await parallelWorker.StartWorkItem
                        (
                            async workerId => {
                            try {
                                await processAsync(new ForEachInfo <T>(itemValue, 0, workerId, ec.CancellationToken));
                            }
                            catch (Exception exc) {
                                ec.Add(exc);
                            }
                            finally {
                                idleDetector.Leave();
                            }
                        },
                            ec.CancellationToken
                        );
                    }
                }
                finally {
                    await idleDetector.WaitForIdle(CancellationToken.None);

                    if (onCloseAsync != null)
                    {
                        try {
                            await onCloseAsync();
                        }
                        catch (Exception exc) {
                            ec.Add(exc);
                        }
                    }
                }
            };

            return(t);
        }
Пример #2
0
        public static Func <Task> ParallelForEach <T>(IQueueSource <T>[] sources, InputPriorities inputPriorities, ParallelWorker parallelWorker,
                                                      Func <ForEachInfo <T>, Task> processAsync, Func <Task> onCloseAsync, ExceptionCollector ec)
        {
            Func <Task> t = async delegate {
                IdleDetector idleDetector = new IdleDetector();
                try {
                    int    sourceCount           = sources.Length;
                    bool[] atEof                 = new bool[sourceCount];
                    RoundRobinLoopGenerator loop = new RoundRobinLoopGenerator(sourceCount, inputPriorities);
                    while (!(atEof.All(e => e)))
                    {
                        var ops = Utils.OperationStarters <int, Option <T> >();

                        loop.ForEach
                        (
                            j => { ops = ops.AddIf(!atEof[j], j, Utils.StartableGet <T, Option <T> >(sources[j], a => new Some <T>(a), new None <T>())); }
                        );

                        Tuple <int, Option <T> > result = await ops.CompleteAny(ec.CancellationToken);

                        if (result.Item2.HasValue)
                        {
                            int sourceIndex = result.Item1;
                            T   itemValue   = result.Item2.Value;

                            idleDetector.Enter();

                            await parallelWorker.StartWorkItem
                            (
                                async workerId => {
                                try {
                                    await processAsync(new ForEachInfo <T>(itemValue, sourceIndex, workerId, ec.CancellationToken));
                                }
                                catch (Exception exc) {
                                    ec.Add(exc);
                                }
                                finally {
                                    idleDetector.Leave();
                                }
                            },
                                ec.CancellationToken
                            );
                        }
                        else
                        {
                            atEof[result.Item1] = true;
                        }
                    }
                }
                finally {
                    if (onCloseAsync != null)
                    {
                        try {
                            await onCloseAsync();
                        }
                        catch (Exception exc) {
                            ec.Add(exc);
                        }
                    }
                }
            };

            return(t);
        }
Пример #3
0
        /// <summary>
        /// OrderedParallelSelect
        /// </summary>
        /// <param name="queue"></param>
        /// <param name="parallelWorker"></param>
        /// <param name="func"></param>
        /// <param name="capacity"></param>
        /// <typeparam name="T"></typeparam>
        /// <typeparam name="U"></typeparam>
        /// <returns></returns>
        public static IQueueSource <U> OrderedParallelSelect <T, U>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <U> > func, int?capacity = null)
        {
            Func <Tuple <int, T>, Task <Tuple <int, U> > > func2 = async pair => new Tuple <int, U>(pair.Item1, await func(pair.Item2));

            return(queue.Indexed().ParallelSelect(parallelWorker, func2, capacity).Reorder(pair => pair.Item1, 0, capacity).SynchronousSelect(pair => pair.Item2));
        }
Пример #4
0
        /// <summary>
        /// OrderedParallelWhere
        /// </summary>
        /// <param name="queue"></param>
        /// <param name="parallelWorker"></param>
        /// <param name="predicate"></param>
        /// <param name="capacity"></param>
        /// <typeparam name="T"></typeparam>
        /// <returns></returns>
        public static IQueueSource <T> OrderedParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null)
        {
            Func <Tuple <int, T>, Task <Tuple <int, T, bool> > > func2 = async pair => new Tuple <int, T, bool>(pair.Item1, pair.Item2, await predicate(pair.Item2));

            return(queue.Indexed().ParallelSelect(parallelWorker, func2, capacity).Reorder(pair => pair.Item1, 0, capacity).Where(i => Task.FromResult(i.Item3), capacity)
                   .SynchronousSelect(i => i.Item2));
        }
Пример #5
0
        /// <summary>
        /// ParallelWhere
        /// </summary>
        /// <param name="queue"></param>
        /// <param name="parallelWorker"></param>
        /// <param name="predicate"></param>
        /// <param name="capacity"></param>
        /// <typeparam name="T"></typeparam>
        /// <returns></returns>
        public static IQueueSource <T> ParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null)
        {
            AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2);

            IdleDetector idleDetector = new IdleDetector();

            Func <Task> workPoster = async delegate {
                while (true)
                {
                    Optional <T> item = await queue.Dequeue(CancellationToken.None);

                    if (!item.HasValue)
                    {
                        break;
                    }
                    T itemValue = item.Value;

                    idleDetector.Enter();

                    Task doingWork = await parallelWorker.StartWorkItem
                                     (
                        async (int workerId) => {
                        try {
                            if (await predicate(itemValue))
                            {
                                await outQueue.Enqueue(itemValue, CancellationToken.None);
                            }
                        }
                        finally {
                            idleDetector.Leave();
                        }
                    },
                        CancellationToken.None
                                     );
                }

                await idleDetector.WaitForIdle(CancellationToken.None);

                outQueue.WriteEof();
            };

            Task.Run(workPoster);

            return(outQueue);
        }