public static void ComputeInParallel(IList <TData> baseData, Func <TData, TIntermediate> parallelWork, Action <TIntermediate> recombination, int numberOfPartitions)
{
var intermediateResults = new BlockingCollection <ProcessedPartition>();
ThreadParallel.Invoke(() =>
{
try
{
var partitionSize = (int)Math.Ceiling(baseData.Count / (float)numberOfPartitions);
ThreadParallel.ForEach(baseData.Select((d, i) => new TaggedBaseData(d, i)).GroupBy(d => d.TaskNumber / partitionSize), g =>
{
intermediateResults.Add(new ProcessedPartition(g.Select(d => parallelWork(d.Data)), g.Key));
});
}
finally
{
intermediateResults.CompleteAdding();
}
}, () =>
{
int expecting = 0;
var backlog = new Dictionary <int, IEnumerable <TIntermediate> >();
foreach (var group in intermediateResults.GetConsumingEnumerable())
{
if (group.TaskNumber != expecting)
{
// if we are not ready for this yet add it to the backlog
backlog[group.TaskNumber] = group.ProcessedData;
continue;
}
IEnumerable <TIntermediate> toProcess = group.ProcessedData;
while (true)
{
foreach (var element in toProcess)
{
recombination(element);
}
expecting++;
// now see if we can combine with the backlog
if (!backlog.TryGetValue(expecting, out toProcess))
{
// if we can't find the next task wait for another
// task to finish
break;
}
else
{
backlog.Remove(expecting);
}
}
}
});
}
public static void ComputeInParallel(IList <TData> baseData, Func <TData, int, TIntermediate> parallelWork, Action <TIntermediate, int> recombination)
{
var intermediateResults = new BlockingCollection <TaggedIntermediate>();
ThreadParallel.Invoke(() =>
{
try
{
ThreadParallel.ForEach(baseData.Select((d, i) => new TaggedBaseData(d, i)), d =>
{
intermediateResults.Add(new TaggedIntermediate(parallelWork(d.Data, d.TaskNumber), d.TaskNumber));
});
}
finally
{
intermediateResults.CompleteAdding();
}
}, () =>
{
int expecting = 0;
var backlog = new Dictionary <int, TIntermediate>();
foreach (var newData in intermediateResults.GetConsumingEnumerable())
{
if (newData.TaskNumber != expecting)
{
// if we are not ready for this yet add it to the backlog
backlog[newData.TaskNumber] = newData.ProcessedData;
continue;
}
TIntermediate toProcess = newData.ProcessedData;
while (true)
{
recombination(toProcess, expecting);
expecting++;
// now see if we can combine with the backlog
if (!backlog.TryGetValue(expecting, out toProcess))
{
// if we can't find the next task wait for another
// task to finish
break;
}
else
{
backlog.Remove(expecting);
}
}
}
});
}
/// <summary>
/// Executes each of the provided actions, possibly in parallel, unless the operation is cancelled by the user.
/// </summary>
public static void Invoke(SystemParallelOptions parallelOptions, params Action[] actions)
{
ExceptionProvider.ThrowUncontrolledInvocationException(nameof(SystemParallel.Invoke));
SystemParallel.Invoke(parallelOptions, actions);
}