public static Func <Task> ParallelForEach <T>(IQueueSource <T> source, ParallelWorker parallelWorker, Func <ForEachInfo <T>, Task> processAsync, Func <Task> onCloseAsync, ExceptionCollector ec)
{
Func <Task> t = async delegate()
{
IdleDetector idleDetector = new IdleDetector();
try
{
while (true)
{
var item = await source.Dequeue(ec.CancellationToken);
if (!item.HasValue)
{
break;
}
T itemValue = item.Value;
idleDetector.Enter();
await parallelWorker.StartWorkItem
(
async workerId =>
{
try
{
await processAsync(new ForEachInfo <T>(itemValue, 0, workerId, ec.CancellationToken));
}
catch (Exception exc)
{
ec.Add(exc);
}
finally
{
idleDetector.Leave();
}
},
ec.CancellationToken
);
}
}
finally
{
await idleDetector.WaitForIdle(CancellationToken.None);
if (onCloseAsync != null)
{
try
{
await onCloseAsync();
}
catch (Exception exc)
{
ec.Add(exc);
}
}
}
};
return(t);
}
public void OrderedParallelPipelineTest()
{
Random r = new Random((int)((System.Diagnostics.Stopwatch.GetTimestamp() >> 3) & 0x7FFFFFFF));
int COUNT = 64;
int[] delays = new int[COUNT];
int[] delays2 = new int[COUNT];
foreach (int i in Enumerable.Range(0, COUNT))
{
delays[i] = 50 + r.Next(200) + (r.Next(5) == 0 ? 500 : 0);
delays2[i] = 50 + r.Next(200) + (r.Next(5) == 0 ? 500 : 0);
}
Func<int, Task<long>> proc = async delegate (int w)
{
System.Diagnostics.Debug.WriteLine($"proc begin wait for {w}");
await Task.Delay(delays[w]);
System.Diagnostics.Debug.WriteLine($"proc end wait for {w}");
return (long)w;
};
Func<long, Task<bool>> predicate = async delegate (long l)
{
System.Diagnostics.Debug.WriteLine($"predicate begin wait for {l}");
await Task.Delay(delays[(int)l]);
System.Diagnostics.Debug.WriteLine($"predicate end wait for {l}");
return (l & 2L) == 0L;
};
ParallelWorker pWorker = new ParallelWorker(3);
var x = Enumerable.Range(0, COUNT)
.AsQueueSource(5)
.OrderedParallelSelect(pWorker, proc, 5)
.OrderedParallelWhere(pWorker, predicate, 5)
.AsEnumerable();
int actualCount = 0;
long? oldI = null;
foreach (long i in x)
{
++actualCount;
System.Diagnostics.Debug.WriteLine($"Received {i}");
Assert.IsTrue(!oldI.HasValue || oldI.Value < i);
oldI = i;
}
Assert.AreEqual(COUNT / 2, actualCount);
}
public static Func <Task> ParallelForEach <T>(IQueueSource <T>[] sources, InputPriorities inputPriorities, ParallelWorker parallelWorker, Func <ForEachInfo <T>, Task> processAsync, Func <Task> onCloseAsync, ExceptionCollector ec)
{
Func <Task> t = async delegate()
{
IdleDetector idleDetector = new IdleDetector();
try
{
int sourceCount = sources.Length;
bool[] atEof = new bool[sourceCount];
RoundRobinLoopGenerator loop = new RoundRobinLoopGenerator(sourceCount, inputPriorities);
while (!(atEof.All(e => e)))
{
var ops = Utils.OperationStarters <int, Option <T> >();
loop.ForEach
(
j => { ops = ops.AddIf(!atEof[j], j, Utils.StartableGet <T, Option <T> >(sources[j], a => new Some <T>(a), new None <T>())); }
);
Tuple <int, Option <T> > result = await ops.CompleteAny(ec.CancellationToken);
if (result.Item2.HasValue)
{
int sourceIndex = result.Item1;
T itemValue = result.Item2.Value;
idleDetector.Enter();
await parallelWorker.StartWorkItem
(
async workerId =>
{
try
{
await processAsync(new ForEachInfo <T>(itemValue, sourceIndex, workerId, ec.CancellationToken));
}
catch (Exception exc)
{
ec.Add(exc);
}
finally
{
idleDetector.Leave();
}
},
ec.CancellationToken
);
}
else
{
atEof[result.Item1] = true;
}
}
}
finally
{
if (onCloseAsync != null)
{
try
{
await onCloseAsync();
}
catch (Exception exc)
{
ec.Add(exc);
}
}
}
};
return(t);
}
public static IQueueSource <T> OrderedParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null)
{
Func <Tuple <int, T>, Task <Tuple <int, T, bool> > > func2 = async pair => new Tuple <int, T, bool>(pair.Item1, pair.Item2, await predicate(pair.Item2));
return(queue.Indexed().ParallelSelect(parallelWorker, func2, capacity).Reorder(pair => pair.Item1, 0, capacity).Where(i => Task.FromResult(i.Item3), capacity).SynchronousSelect(i => i.Item2));
}
public static IQueueSource <U> OrderedParallelSelect <T, U>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <U> > func, int?capacity = null)
{
Func <Tuple <int, T>, Task <Tuple <int, U> > > func2 = async pair => new Tuple <int, U>(pair.Item1, await func(pair.Item2));
return(queue.Indexed().ParallelSelect(parallelWorker, func2, capacity).Reorder(pair => pair.Item1, 0, capacity).SynchronousSelect(pair => pair.Item2));
}
public static IQueueSource <T> ParallelWhere <T>(this IQueueSource <T> queue, ParallelWorker parallelWorker, Func <T, Task <bool> > predicate, int?capacity = null)
{
AsyncQueue <T> outQueue = new AsyncQueue <T>(capacity ?? 2);
IdleDetector idleDetector = new IdleDetector();
Func <Task> workPoster = async delegate()
{
while (true)
{
Option <T> item = await queue.Dequeue(CancellationToken.None);
if (!item.HasValue)
{
break;
}
T itemValue = item.Value;
idleDetector.Enter();
Task doingWork = await parallelWorker.StartWorkItem
(
async (int workerId) =>
{
try
{
if (await predicate(itemValue))
{
await outQueue.Enqueue(itemValue, CancellationToken.None);
}
}
finally
{
idleDetector.Leave();
}
},
CancellationToken.None
);
}
await idleDetector.WaitForIdle(CancellationToken.None);
outQueue.WriteEof();
};
Task.Run(workPoster);
return(outQueue);
}