internal override IList <IEnumerable <KeyValuePair <long, TSource> > > GetOrderedEnumerables(QueryOptions options)
{
ReverseList <TSource> reversed = new ReverseList <TSource> (source.ToListOrdered());
OrderablePartitioner <TSource> partitioner = ParallelPartitioner.CreateForStrips(reversed, 1);
return(WrapHelper.Wrap(partitioner.GetOrderablePartitions(options.PartitionCount)));
}
internal override IList <IEnumerable <KeyValuePair <long, T> > > GetOrderedEnumerables(QueryOptions options)
{
OrderablePartitioner <T> partitioner = null;
if (customPartitioner != null)
{
partitioner = customPartitioner as OrderablePartitioner <T>;
if (partitioner == null)
{
throw new InvalidOperationException("The partitionner you are using doesn't support ordered partitionning");
}
}
else
{
partitioner =
(options.UseStrip) ? ParallelPartitioner.CreateForStrips(source, 1) : ParallelPartitioner.CreateBest(source);
}
options.PartitionerSettings = Tuple.Create(partitioner.KeysOrderedAcrossPartitions,
partitioner.KeysOrderedInEachPartition,
partitioner.KeysNormalized);
// We only support one style of partitioning at the moment.
// Standard partitioners follow this style.
if (options.UseStrip && (!partitioner.KeysOrderedInEachPartition || partitioner.KeysOrderedAcrossPartitions))
{
throw new NotImplementedException("Partitioner must have KeysOrderedInEachPartition "
+ "and !KeysOrderedAcrossPartitions"
+ "to be used with indexed operators");
}
return(WrapHelper.Wrap(partitioner.GetOrderablePartitions(options.PartitionCount)));
}
private static bool RunTestWithAlgorithm(int dataSize, int partitionCount, int algorithm)
{
//we set up the KeyValuePair in the way that keys and values should always be the same
//for all partitioning algorithms. So that we can use a bitmap (boolarray) to check whether
//any elements are missing in the end.
int[] data = new int[dataSize];
for (int i = 0; i < dataSize; i++)
{
data[i] = i;
}
bool passed = true;
IEnumerator <KeyValuePair <long, int> >[] partitionsUnderTest = new IEnumerator <KeyValuePair <long, int> > [partitionCount];
//step 1: test GetOrderablePartitions
DebugMessage(false, () => Console.WriteLine("Testing GetOrderablePartitions"));
OrderablePartitioner <int> partitioner = PartitioningWithAlgorithm <int>(data, algorithm);
var partitions1 = partitioner.GetOrderablePartitions(partitionCount);
//convert it to partition array for testing
for (int i = 0; i < partitionCount; i++)
{
partitionsUnderTest[i] = partitions1[i];
}
Assert(partitions1.Count == partitionCount, "RunPartitionerStaticTest_LoadBalanceIList");
passed &= TestPartitioningCore(dataSize, partitionCount, data, IsStaticPartition(algorithm), partitionsUnderTest);
//step 2: test GetOrderableDynamicPartitions
DebugMessage(false, () => Console.WriteLine("Testing GetOrderableDynamicPartitions"));
bool gotException = false;
try
{
var partitions2 = partitioner.GetOrderableDynamicPartitions();
for (int i = 0; i < partitionCount; i++)
{
partitionsUnderTest[i] = partitions2.GetEnumerator();
}
passed &= TestPartitioningCore(dataSize, partitionCount, data, IsStaticPartition(algorithm), partitionsUnderTest);
}
catch (NotSupportedException)
{
//swallow this exception: static partitioning doesn't support GetOrderableDynamicPartitions
gotException = true;
}
if (IsStaticPartition(algorithm) && !gotException)
{
TestHarness.TestLog("Failure: didn't catch \"NotSupportedException\" for static partitioning");
passed = false;
}
return(passed);
}
internal override IList <IEnumerable <KeyValuePair <long, T> > > GetOrderedEnumerables(QueryOptions options)
{
int partitionCount;
IList <T> aggregList = GetAggregatedList(out partitionCount);
IList <T> result = ParallelQuickSort <T> .Sort(aggregList, comparison);
OrderablePartitioner <T> partitioner = ParallelPartitioner.CreateForStrips(result, 1);
return(WrapHelper.Wrap(partitioner.GetOrderablePartitions(options.PartitionCount)));
}
private static void RunTestWithAlgorithm(int dataSize, int partitionCount, int algorithm)
{
//we set up the KeyValuePair in the way that keys and values should always be the same
//for all partitioning algorithms. So that we can use a bitmap (boolarray) to check whether
//any elements are missing in the end.
int[] data = new int[dataSize];
for (int i = 0; i < dataSize; i++)
{
data[i] = i;
}
IEnumerator <KeyValuePair <long, int> >[] partitionsUnderTest = new IEnumerator <KeyValuePair <long, int> > [partitionCount];
//step 1: test GetOrderablePartitions
OrderablePartitioner <int> partitioner = PartitioningWithAlgorithm <int>(data, algorithm);
var partitions1 = partitioner.GetOrderablePartitions(partitionCount);
//convert it to partition array for testing
for (int i = 0; i < partitionCount; i++)
{
partitionsUnderTest[i] = partitions1[i];
}
if (partitions1.Count != partitionCount)
{
Assert.False(true, String.Format(
"RunPartitionerStaticTest_LoadBalanceIList: FAILED. partitions1.count: {0} != partitioncount: {1}", partitions1.Count, partitionCount));
}
TestPartitioningCore(dataSize, partitionCount, data, IsStaticPartition(algorithm), partitionsUnderTest);
//step 2: test GetOrderableDynamicPartitions
bool gotException = false;
try
{
var partitions2 = partitioner.GetOrderableDynamicPartitions();
for (int i = 0; i < partitionCount; i++)
{
partitionsUnderTest[i] = partitions2.GetEnumerator();
}
TestPartitioningCore(dataSize, partitionCount, data, IsStaticPartition(algorithm), partitionsUnderTest);
}
catch (NotSupportedException)
{
//swallow this exception: static partitioning doesn't support GetOrderableDynamicPartitions
gotException = true;
}
if (IsStaticPartition(algorithm) && !gotException)
{
Assert.False(true, "RunPartitionerStaticTest_LoadBalanceIList: Failure: didn't catch \"NotSupportedException\" for static partitioning");
}
}
public void PartitionerCreateLongWithExplicitRange()
{
OrderablePartitioner <Tuple <long, long> > partitioner = Partitioner.Create((long)1, (long)20, (long)5);
var partitions = partitioner.GetOrderablePartitions(3);
Assert.AreEqual(3, partitions.Count);
Assert.That(partitions, Is.All.Not.Null);
var iterator = partitions[0];
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(CreateL(0, 1, 6)));
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(CreateL(1, 6, 11)));
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(CreateL(2, 11, 16)));
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(CreateL(3, 16, 20)));
Assert.IsFalse(partitions[1].MoveNext());
Assert.IsFalse(partitions[2].MoveNext());
}
public void PartitionerCreateIntegerWithExplicitRange()
{
OrderablePartitioner <Tuple <int, int> > partitioner = Partitioner.Create(1, 20, 5);
var partitions = partitioner.GetOrderablePartitions(3);
Assert.AreEqual(3, partitions.Count);
CollectionAssert.AllItemsAreNotNull(partitions);
var iterator = partitions[0];
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(Create(0, 1, 6)));
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(Create(1, 6, 11)));
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(Create(2, 11, 16)));
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(Create(3, 16, 20)));
Assert.IsFalse(partitions[1].MoveNext());
Assert.IsFalse(partitions[2].MoveNext());
}
public void PartitionerCreateIntegerWithExplicitRange()
{
OrderablePartitioner <Tuple <int, int> > partitioner = Partitioner.Create(1, 20, 5);
var partitions = partitioner.GetOrderablePartitions(3);
Assert.AreEqual(3, partitions.Count);
partitions.AreIsAllNotNull();
//--TODO Assert.That (partitions, Is.All.Not.Null);
var iterator = partitions[0];
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(Create(0, 1, 6)));
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(Create(1, 6, 11)));
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(Create(2, 11, 16)));
Assert.IsTrue(iterator.MoveNext());
Assert.IsTrue(iterator.Current.Equals(Create(3, 16, 20)));
Assert.IsFalse(partitions[1].MoveNext());
Assert.IsFalse(partitions[2].MoveNext());
}
internal override void GivePartitionedStream(IPartitionedStreamRecipient <TElement> recipient)
{
Contract.Assert(m_settings.DegreeOfParallelism.HasValue);
int partitionCount = m_settings.DegreeOfParallelism.Value;
OrderablePartitioner <TElement> orderablePartitioner = m_partitioner as OrderablePartitioner <TElement>;
// If the partitioner is not orderable, it will yield zeros as order keys. The order index state
// is irrelevant.
OrdinalIndexState indexState = (orderablePartitioner != null)
? GetOrdinalIndexState(orderablePartitioner)
: OrdinalIndexState.Shuffled;
PartitionedStream <TElement, int> partitions = new PartitionedStream <TElement, int>(
partitionCount,
Util.GetDefaultComparer <int>(),
indexState);
if (orderablePartitioner != null)
{
IList <IEnumerator <KeyValuePair <long, TElement> > > partitionerPartitions =
orderablePartitioner.GetOrderablePartitions(partitionCount);
if (partitionerPartitions == null)
{
throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_NullPartitionList));
}
if (partitionerPartitions.Count != partitionCount)
{
throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_WrongNumberOfPartitions));
}
for (int i = 0; i < partitionCount; i++)
{
IEnumerator <KeyValuePair <long, TElement> > partition = partitionerPartitions[i];
if (partition == null)
{
throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_NullPartition));
}
partitions[i] = new OrderablePartitionerEnumerator(partition);
}
}
else
{
IList <IEnumerator <TElement> > partitionerPartitions =
m_partitioner.GetPartitions(partitionCount);
if (partitionerPartitions == null)
{
throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_NullPartitionList));
}
if (partitionerPartitions.Count != partitionCount)
{
throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_WrongNumberOfPartitions));
}
for (int i = 0; i < partitionCount; i++)
{
IEnumerator <TElement> partition = partitionerPartitions[i];
if (partition == null)
{
throw new InvalidOperationException(SR.GetString(SR.PartitionerQueryOperator_NullPartition));
}
partitions[i] = new PartitionerEnumerator(partition);
}
}
recipient.Receive <int>(partitions);
}
internal override void GivePartitionedStream(IPartitionedStreamRecipient <TElement> recipient)
{
int partitionCount = this.m_settings.DegreeOfParallelism.Value;
OrderablePartitioner <TElement> partitioner = this.m_partitioner as OrderablePartitioner <TElement>;
OrdinalIndexState indexState = (partitioner != null) ? PartitionerQueryOperator <TElement> .GetOrdinalIndexState(partitioner) : OrdinalIndexState.Shuffled;
PartitionedStream <TElement, int> partitionedStream = new PartitionedStream <TElement, int>(partitionCount, Util.GetDefaultComparer <int>(), indexState);
if (partitioner != null)
{
IList <IEnumerator <KeyValuePair <long, TElement> > > orderablePartitions = partitioner.GetOrderablePartitions(partitionCount);
if (orderablePartitions == null)
{
throw new InvalidOperationException(System.Linq.SR.GetString("PartitionerQueryOperator_NullPartitionList"));
}
if (orderablePartitions.Count != partitionCount)
{
throw new InvalidOperationException(System.Linq.SR.GetString("PartitionerQueryOperator_WrongNumberOfPartitions"));
}
for (int i = 0; i < partitionCount; i++)
{
IEnumerator <KeyValuePair <long, TElement> > sourceEnumerator = orderablePartitions[i];
if (sourceEnumerator == null)
{
throw new InvalidOperationException(System.Linq.SR.GetString("PartitionerQueryOperator_NullPartition"));
}
partitionedStream[i] = new PartitionerQueryOperator <TElement> .OrderablePartitionerEnumerator(sourceEnumerator);
}
}
else
{
IList <IEnumerator <TElement> > partitions = this.m_partitioner.GetPartitions(partitionCount);
if (partitions == null)
{
throw new InvalidOperationException(System.Linq.SR.GetString("PartitionerQueryOperator_NullPartitionList"));
}
if (partitions.Count != partitionCount)
{
throw new InvalidOperationException(System.Linq.SR.GetString("PartitionerQueryOperator_WrongNumberOfPartitions"));
}
for (int j = 0; j < partitionCount; j++)
{
IEnumerator <TElement> enumerator2 = partitions[j];
if (enumerator2 == null)
{
throw new InvalidOperationException(System.Linq.SR.GetString("PartitionerQueryOperator_NullPartition"));
}
partitionedStream[j] = new PartitionerQueryOperator <TElement> .PartitionerEnumerator(enumerator2);
}
}
recipient.Receive <int>(partitionedStream);
}