internal static void ParallelForEach <T>(IEnumerable <T> objects, int threads, Action <int, T> action, CancelRequest cancel = null) { ExecuteInParallelBatches(objects, threads, 1, (thread, batch) => { action(thread, batch.Item2.First()); }, null, cancel); }
public static void ExecuteInParallelBatches <T>(IEnumerable <T> objects, int threads, int batchSize, Action <int, Tuple <int, IList <T> > > action, int?queueCapacity = null, CancelRequest cancel = null) { if (cancel == null) { cancel = new CancelRequest(); } using (var queue = queueCapacity == null ? new BlockingCollection <Tuple <int, IList <T> > >() : new BlockingCollection <Tuple <int, IList <T> > >(queueCapacity.Value)) { Action <int> workerAction = (thread) => { while (!cancel.IsSet && !queue.IsCompleted) { try { if (queue.TryTake(out var batch, 1000)) { if (!cancel.IsSet) { action(thread, batch); } } } catch (Exception) { cancel.Set(); queue.CompleteAdding(); // This is to let main thread to throw if it is blocked on write by limited queue capacity throw; } } if (cancel.IsSet) { queue.CompleteAdding(); // This is to let main thread to throw if it is blocked on write by limited queue capacity in case of external cancel } }; var workers = new List <Task> { StartTask(() => workerAction(0)) }; try { var batchId = 0; var batchList = new List <T>(); foreach (var obj in objects) { if (cancel.IsSet) { break; } batchList.Add(obj); if (batchList.Count == batchSize) { AddToQueueWithTrapping(queue, batchId, batchList, cancel, workers, threads, workerAction); batchList = new List <T>(); batchId++; } } if (!cancel.IsSet && batchList.Count > 0) { AddToQueueWithTrapping(queue, batchId, batchList, cancel, workers, threads, workerAction); } queue.CompleteAdding(); Task.WaitAll(workers.ToArray()); } catch (Exception) { queue.CompleteAdding(); cancel.Set(); throw; } } }
private static void AddToQueueWithTrapping <T>(BlockingCollection <Tuple <int, IList <T> > > queue, int batchId, IList <T> batchList, CancelRequest cancel, ICollection <Task> workers, int maxWorkers, Action <int> workerAction) { try { queue.Add(new Tuple <int, IList <T> >(batchId, batchList)); if (workers.Count < maxWorkers && queue.Count > 0) { var id = workers.Count; workers.Add(StartTask(() => workerAction(id))); } } catch (Exception e) { if (cancel.IsSet && IsCleanupQueueException(e)) { return; } throw; } }