public MonotonicSubscriptionWallClock(
IObservable <TPayload> observable,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy onCompletedPolicy,
TimelinePolicy timelinePolicy)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Adjust(),
flushPolicy,
punctuationPolicy,
onCompletedPolicy,
null)
{
if (timelinePolicy.punctuationInterval > default(TimeSpan))
{
this.timer = new Timer(EmitPunctuation, null, new TimeSpan(0), timelinePolicy.punctuationInterval);
}
}
public IntervalArraySubscriptionThrowNone(
IObservable <ArraySegment <TPayload> > observable,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Throw(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.None(),
onCompletedPolicy,
diagnosticOutput)
{
this.startEdgeExtractor = startEdgeExtractor;
this.startEdgeFunction = startEdgeExtractor.Compile();
this.endEdgeExtractor = endEdgeExtractor;
this.endEdgeFunction = endEdgeExtractor.Compile();
}
public CompiledPartitionedAfaPipe_MultiEventList(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
: base(stream, observer, afa, maxDuration, false)
{
this.getPartitionKey = GetPartitionExtractor <TPartitionKey, TKey>();
this.activeStates = new FastMap <GroupedActiveState <TKey, TRegister> >();
this.activeFindTraverser = new FastMap <GroupedActiveState <TKey, TRegister> > .FindTraverser(this.activeStates);
}
public PartitionedIntervalArraySubscriptionThrowNone(
IObservable <ArraySegment <TPayload> > observable,
Expression <Func <TPayload, TKey> > partitionExtractor,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
string identifier,
Streamable <PartitionKey <TKey>, TPayload> streamable,
IStreamObserver <PartitionKey <TKey>, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Throw(),
PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy.None(),
PeriodicLowWatermarkPolicy.None(),
onCompletedPolicy,
diagnosticOutput)
{
this.partitionExtractor = partitionExtractor;
this.partitionFunction = partitionExtractor.Compile();
this.startEdgeExtractor = startEdgeExtractor;
this.startEdgeFunction = startEdgeExtractor.Compile();
this.endEdgeExtractor = endEdgeExtractor;
this.endEdgeFunction = endEdgeExtractor.Compile();
}
public UnionPipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer)
: base(stream, observer)
{
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
this.pool.Get(out this.output);
this.output.Allocate();
}
public CompiledUngroupedAfaPipe_EventList(Streamable <Empty, TRegister> stream, IStreamObserver <Empty, TRegister> observer, object afa, long maxDuration)
: base(stream, observer, afa, maxDuration, false)
{
this.activeStates = new FastLinkedList <GroupedActiveState <Empty, TRegister> >();
this.activeStatesTraverser = new FastLinkedList <GroupedActiveState <Empty, TRegister> > .ListTraverser(this.activeStates);
this.currentList = new List <TPayload>();
this.lastSyncTime = -1;
}
public PartitionedExtendLifetimePipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long duration)
: base(stream, observer)
{
this.duration = duration;
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
this.errorMessages = stream.ErrorMessages;
this.pool.Get(out this.output);
this.output.Allocate();
}
public static PartitionedObserverSubscriptionBase <TKey, ArraySegment <PartitionedStreamEvent <TKey, TPayload> >, TPayload, TPayload> CreateSubscription(
IObservable <ArraySegment <PartitionedStreamEvent <TKey, TPayload> > > observable,
string identifier,
Streamable <PartitionKey <TKey>, TPayload> streamable,
IStreamObserver <PartitionKey <TKey>, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput)
{
return(new PartitionedStreamEventArraySubscriptionThrowNone <TKey, TPayload>(observable, identifier, streamable, observer, onCompletedPolicy, diagnosticOutput));
}
public static ObserverSubscriptionBase <ArraySegment <TPayload>, TPayload, TPayload> CreateSubscription(
IObservable <ArraySegment <TPayload> > observable,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput)
{
return(new IntervalArraySubscriptionThrowNone <TPayload>(observable, startEdgeExtractor, endEdgeExtractor, identifier, streamable, observer, onCompletedPolicy, diagnosticOutput));
}
public CompiledPartitionedAfaPipe_SingleEvent(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
: base(stream, observer, afa, maxDuration)
{
if (!this.IsSyncTimeSimultaneityFree)
{
var comparer = stream.Properties.KeyEqualityComparer;
var equalsFunc = comparer.GetEqualsExpr().Compile();
var getHashCodeFunc = comparer.GetGetHashCodeExpr().Compile();
this.seenEvent = comparer.CreateFastDictionary2Generator <TKey, byte>(10, equalsFunc, getHashCodeFunc, stream.Properties.QueryContainer).Invoke();
this.tentativeOutput = new FastDictionary2 <TPartitionKey, FastMap <OutputEvent <TKey, TRegister> > >();
}
}
public CompiledUngroupedAfaPipe(Streamable <Empty, TRegister> stream, IStreamObserver <Empty, TRegister> observer, object afa, long maxDuration)
: base(stream, observer, afa, maxDuration)
{
this.activeStates = new FastLinkedList <GroupedActiveState <Empty, TRegister> >();
if (!this.IsSyncTimeSimultaneityFree)
{
this.seenEvent = 0;
this.tentativeOutput = new FastLinkedList <OutputEvent <Empty, TRegister> >();
this.lastSyncTime = -1;
}
}
public PartitionedQuantizeLifetimePipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long width, long skip, long progress, long offset)
: base(stream, observer)
{
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
this.errorMessages = stream.ErrorMessages;
this.pool.Get(out this.output);
this.output.Allocate();
this.width = width;
this.skip = skip;
this.progress = progress;
this.offset = offset;
}
public CompiledGroupedAfaPipe_MultiEvent(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
: base(stream, observer, afa, maxDuration)
{
this.activeStates = new FastMap <GroupedActiveStateAccumulator <TKey, TPayload, TRegister, TAccumulator> >(1);
this.activeFindTraverser = new FastMap <GroupedActiveStateAccumulator <TKey, TPayload, TRegister, TAccumulator> > .FindTraverser(this.activeStates);
this.keyHeads = new FastMap <TKey>(1);
this.keyHeadsFindTraverser = new FastMap <TKey> .FindTraverser(this.keyHeads);
this.keyHeadsVisibleTraverser = new FastMap <TKey> .VisibleTraverser(this.keyHeads);
this.lastSyncTime = -1;
}
public CompiledGroupedAfaPipe_MultiEventList(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
: base(stream, observer, afa, maxDuration)
{
this.activeStates = new FastMap <GroupedActiveState <TKey, TRegister> >();
this.activeFindTraverser = new FastMap <GroupedActiveState <TKey, TRegister> > .FindTraverser(this.activeStates);
this.currentTimestampEventList = new FastMap <SavedEventList <TKey, TPayload> >();
this.eventListTraverser = new FastMap <SavedEventList <TKey, TPayload> > .FindTraverser(this.currentTimestampEventList);
this.allEventListTraverser = new FastMap <SavedEventList <TKey, TPayload> > .VisibleTraverser(this.currentTimestampEventList);
this.lastSyncTime = -1;
}
public StreamMessageIngressSubscription(
IObservable <StreamMessage <Empty, TPayload> > observable,
string identifier,
Streamable <Empty, TPayload> stream,
IStreamObserver <Empty, TPayload> observer)
: base(
observable,
identifier,
stream,
observer,
DisorderPolicy.Drop(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.None(),
OnCompletedPolicy.None,
null)
{
}
public MonotonicArraySubscriptionSequence(
IObservable <ArraySegment <TPayload> > observable,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
TimelinePolicy timelinePolicy)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Throw(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.None(),
onCompletedPolicy,
null)
=> this.eventsPerSample = timelinePolicy.sampleSize;
public StreamEventArraySubscriptionThrowNone(
IObservable <ArraySegment <StreamEvent <TPayload> > > observable,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Throw(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.None(),
onCompletedPolicy,
diagnosticOutput)
{
}
public BinaryIngressReader(
string identifier,
Streamable <TKey, TPayload> streamable,
IStreamObserver <TKey, TPayload> observer,
int numMessages,
Stream stream,
IIngressScheduler scheduler,
bool delayed)
: base(identifier, streamable, observer)
{
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>();
this.numMessages = numMessages;
this.stream = stream;
this.scheduler = scheduler;
if (!delayed)
{
this.subscription.Enable();
}
}
public PartitionedStreamEventArraySubscriptionThrowNone(
IObservable <ArraySegment <PartitionedStreamEvent <TKey, TPayload> > > observable,
string identifier,
Streamable <PartitionKey <TKey>, TPayload> streamable,
IStreamObserver <PartitionKey <TKey>, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Throw(),
PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy.None(),
PeriodicLowWatermarkPolicy.None(),
onCompletedPolicy,
diagnosticOutput)
{
}
public MonotonicSubscriptionSequence(
IObservable <TPayload> observable,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy onCompletedPolicy,
TimelinePolicy timelinePolicy)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Adjust(),
flushPolicy,
punctuationPolicy,
onCompletedPolicy,
null)
{
this.eventsPerSample = timelinePolicy.sampleSize;
this.currentTime = 0;
}
public DisjointUnionPipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer)
: base(stream, observer)
{
}
public override IDisposable Subscribe(IStreamObserver <TMapKey, TOutput> observer)
{
// asymmetric mapper implies that we have to have a 2-input mapper
Contract.Assert((!this.leftAsymmetric1) || (this.sourceRight1 != null));
Contract.Assert((!this.leftAsymmetric2) || (this.sourceRight2 != null));
var mapArity = Config.MapArity;
var reduceArity = Config.ReduceArity;
// process mapper #1
Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>[] shuffleL2Results1;
if (this.sourceRight1 != null) // two-input mapper
{
// DEAD
shuffleL2Results1 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> [reduceArity];
// [1] spray batches into L1 physical cores
var importLeft1 = new SprayGroupImportStreamable <TMapKey, TMapInputLeft1>(this.sourceLeft1, mapArity, this.leftAsymmetric1);
var importRight1 = new SprayGroupImportStreamable <TMapKey, TMapInputRight1>(this.sourceRight1, mapArity);
// [2] perform the spray lambda on each L1 core
var sprayResults1 = new BinaryMulticastStreamable <TMapKey, TMapInputLeft1, TMapInputRight1, TReduceInput1> [mapArity];
for (int i = 0; i < mapArity; i++)
{
sprayResults1[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft1, TMapInputRight1, TReduceInput1>(importLeft1, importRight1, this.mapper1);
}
// [3] apply shuffle on the result of each spray
Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>[] shuffleL1Results1 = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey> [mapArity];
for (int i = 0; i < mapArity; i++)
{
shuffleL1Results1[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey>(sprayResults1[i], this.keySelector1, reduceArity, i);
}
// [4] Union the shuffled data by group key
MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> .l2index = 0;
for (int i = 0; i < reduceArity; i++)
{
shuffleL2Results1[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>(shuffleL1Results1);
}
}
else // single-input mapper
{
// [1] spray batches into L1 physical cores
var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft1>
(this.sourceLeft1, mapArity, false, this.sprayComparer1);
// [2] perform the spray lambda on each L1 core
var sprayResults1 = new MulticastStreamable <TMapKey, TMapInputLeft1, TReduceInput1> [mapArity];
for (int i = 0; i < mapArity; i++)
{
sprayResults1[i] = new MulticastStreamable <TMapKey, TMapInputLeft1, TReduceInput1>(importLeft, a => this.mapper1(a, null));
}
if (this.reduceInMap || (this.reduceOptions == OperationalHint.Asymmetric))
{
shuffleL2Results1 = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> [mapArity];
for (int i = 0; i < mapArity; i++)
{
shuffleL2Results1[i] = new GroupNestedStreamable <TMapKey, TReduceInput1, TReduceKey>(sprayResults1[i], this.keySelector1);
}
}
else
{
shuffleL2Results1 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> [reduceArity];
// [3] apply shuffle on the result of each spray
Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>[] shuffleL1Results1 = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey> [mapArity];
for (int i = 0; i < mapArity; i++)
{
shuffleL1Results1[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey>(sprayResults1[i], this.keySelector1, reduceArity, i);
}
// [4] Union the shuffled data by group key
MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> .l2index = 0;
for (int i = 0; i < reduceArity; i++)
{
shuffleL2Results1[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>(shuffleL1Results1);
}
}
}
// process mapper #2
Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>[] shuffleL2Results2;
if (this.sourceRight2 != null) // two-input mapper
{
// DEAD
shuffleL2Results2 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> [reduceArity];
// [1] spray batches into L1 physical cores
var importLeft2 = new SprayGroupImportStreamable <TMapKey, TMapInputLeft2>(this.sourceLeft2, mapArity, this.leftAsymmetric2);
var importRight2 = new SprayGroupImportStreamable <TMapKey, TMapInputRight2>(this.sourceRight2, mapArity);
// [2] perform the spray lambda on each L1 core
var sprayResults2 = new BinaryMulticastStreamable <TMapKey, TMapInputLeft2, TMapInputRight2, TReduceInput2> [mapArity];
for (int i = 0; i < mapArity; i++)
{
sprayResults2[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft2, TMapInputRight2, TReduceInput2>(importLeft2, importRight2, this.mapper2);
}
// [3] apply shuffle on the result of each spray
Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>[] shuffleL1Results2 = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey> [mapArity];
for (int i = 0; i < mapArity; i++)
{
shuffleL1Results2[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey>(sprayResults2[i], this.keySelector2, reduceArity, i);
}
// [4] Union the shuffled data by group key
MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> .l2index = 0;
for (int i = 0; i < reduceArity; i++)
{
shuffleL2Results2[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>(shuffleL1Results2);
}
}
else // single-input mapper
{
// [1] spray batches into L1 physical cores
var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft2>(this.sourceLeft2, mapArity, this.reduceOptions == OperationalHint.Asymmetric);
// [2] perform the spray lambda on each L1 core
var sprayResults2 = new MulticastStreamable <TMapKey, TMapInputLeft2, TReduceInput2> [mapArity];
for (int i = 0; i < mapArity; i++)
{
sprayResults2[i] = new MulticastStreamable <TMapKey, TMapInputLeft2, TReduceInput2>(importLeft, a => this.mapper2(a, null));
}
if (this.reduceInMap || (this.reduceOptions == OperationalHint.Asymmetric))
{
shuffleL2Results2 = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> [mapArity];
for (int i = 0; i < mapArity; i++)
{
shuffleL2Results2[i] = new GroupNestedStreamable <TMapKey, TReduceInput2, TReduceKey>(sprayResults2[i], this.keySelector2);
}
}
else
{
shuffleL2Results2 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> [reduceArity];
// [3] apply shuffle on the result of each spray
var shuffleL1Results2 = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey> [mapArity];
for (int i = 0; i < mapArity; i++)
{
shuffleL1Results2[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey>(sprayResults2[i], this.keySelector2, reduceArity, i);
}
// [4] Union the shuffled data by group key
MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> .l2index = 0;
for (int i = 0; i < reduceArity; i++)
{
shuffleL2Results2[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>(shuffleL1Results2);
}
}
}
// process 2-input reducer
// [5] perform the apply lambda on each L2 core
var innerResults
= new BinaryMulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1, TReduceInput2, TBind> [shuffleL2Results1.Length];
var ungroupInnerResults
= new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput> [shuffleL2Results1.Length];
for (int i = 0; i < shuffleL2Results1.Length; i++)
{
innerResults[i] = new BinaryMulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1, TReduceInput2, TBind>(shuffleL2Results1[i], shuffleL2Results2[i], this.reducer);
ungroupInnerResults[i] = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput>(this.sourceLeft1.Properties.KeyEqualityComparer, innerResults[i], this.resultSelector);
}
// [6] final single merging union
var union = new MultiUnionStreamable <TMapKey, TOutput>(ungroupInnerResults, false);
return(union.Subscribe(observer));
}
public DisjointUnionPipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer)
: base(stream, observer)
{
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
}
public CompiledPartitionedAfaPipe_MultiEvent(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
: base(stream, observer, afa, maxDuration)
{
this.activeStates = new FastMap <GroupedActiveStateAccumulator <TKey, TPayload, TRegister, TAccumulator> >(1);
this.activeFindTraverser = new FastMap <GroupedActiveStateAccumulator <TKey, TPayload, TRegister, TAccumulator> > .FindTraverser(this.activeStates);
}
public override IDisposable Subscribe(IStreamObserver <TMapKey, TOutput> observer)
{
// asymmetric mapper implies that we have to have a 2-input mapper
Contract.Assert((!this.leftAsymmetric) || (this.sourceRight != null));
var mapArity = this.isMulticore ? 1 : Config.MapArity;
var reduceArity = this.isMulticore ? 1 : Config.ReduceArity;
if (this.keySelector != null)
{
if (this.sourceRight != null) // two-input mapper
{
// [1] spray batches into L1 physical cores
var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity, this.leftAsymmetric);
var importRight = new SprayGroupImportStreamable <TMapKey, TMapInputRight>(this.sourceRight, mapArity);
// [2] perform the spray lambda on each L1 core
var sprayResults = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput> [mapArity];
for (int i = 0; i < mapArity; i++)
{
sprayResults[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput>(importLeft, importRight, this.mapper);
}
// [3] apply shuffle on the result of each spray
Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>[] shuffleL1Results = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey> [mapArity];
for (int i = 0; i < mapArity; i++)
{
shuffleL1Results[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey>(this.keyComparer, sprayResults[i], this.keySelector, reduceArity, i);
}
// [4] Union the shuffled data by group key
MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> .l2index = 0;
var shuffleL2Results = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [reduceArity];
for (int i = 0; i < reduceArity; i++)
{
shuffleL2Results[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>(shuffleL1Results);
}
// [5] perform the apply lambda on each L2 core
var innerResults = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind> [reduceArity];
var ungroupInnerResults = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput> [reduceArity];
for (int i = 0; i < reduceArity; i++)
{
innerResults[i] = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind>(shuffleL2Results[i], this.reducer);
ungroupInnerResults[i] = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput>(this.sourceLeft.Properties.KeyEqualityComparer, innerResults[i], this.resultSelector);
}
// [6] final single merging union
var union = new MultiUnionStreamable <TMapKey, TOutput>(ungroupInnerResults, false);
return(union.Subscribe(observer));
}
else // single-input mapper
{
// [1] spray batches into L1 physical cores
var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity, this.leftAsymmetric, this.sprayComparer);
// [2] perform the spray lambda on each L1 core
var sprayResults = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput> [mapArity];
for (int i = 0; i < mapArity; i++)
{
sprayResults[i] = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput>(importLeft, a => this.mapper(a, null));
}
Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>[] mergeInputs;
if (this.reduceInMap) // apply reducer in map phase itself
{
// [3] apply shuffle on the result of each spray
mergeInputs = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [mapArity];
for (int i = 0; i < mapArity; i++)
{
mergeInputs[i] = new GroupNestedStreamable <TMapKey, TReduceInput, TReduceKey>(this.keyComparer, sprayResults[i], this.keySelector);
}
}
else
{
// [3] apply shuffle on the result of each spray
Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>[] shuffleL1Results = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey> [mapArity];
for (int i = 0; i < mapArity; i++)
{
shuffleL1Results[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey>(this.keyComparer, sprayResults[i], this.keySelector, reduceArity, i);
}
// [4] Union the shuffled data by group key
MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> .l2index = 0;
mergeInputs = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [reduceArity];
mergeInputs = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [reduceArity];
for (int i = 0; i < reduceArity; i++)
{
mergeInputs[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>(shuffleL1Results);
}
}
// [5] perform the apply lambda on each L2 core
var innerResults
= new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind> [mergeInputs.Length];
var ungroupInnerResults
= new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput> [mergeInputs.Length];
for (int i = 0; i < mergeInputs.Length; i++)
{
innerResults[i] = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind>(mergeInputs[i], this.reducer);
ungroupInnerResults[i] = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput>(this.sourceLeft.Properties.KeyEqualityComparer, innerResults[i], this.resultSelector);
}
// [6] final single merging union
var union = new MultiUnionStreamable <TMapKey, TOutput>(ungroupInnerResults, false);
return(union.Subscribe(observer));
}
}
else
{
if (this.sourceRight != null) // two-input mapper
{
// [1] spray batches into L1 physical cores
var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity, this.leftAsymmetric);
var importRight = new SprayGroupImportStreamable <TMapKey, TMapInputRight>(this.sourceRight, mapArity);
// [2] perform the spray lambda on each L1 core
var sprayResults = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput> [mapArity];
for (int i = 0; i < mapArity; i++)
{
sprayResults[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput>(importLeft, importRight, this.mapper);
}
// [4] Union the shuffled data by group key
MultiUnionStreamable <TMapKey, TReduceInput> .l2index = 0;
var shuffleL2Result
= new MultiUnionStreamable <TMapKey, TReduceInput>(sprayResults, false) as MultiUnionStreamable <TMapKey, TOutput>;
return(shuffleL2Result.Subscribe(observer));
}
else // single-input mapper
{
// [1] spray batches into L1 physical cores
var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity);
// [2] perform the spray lambda on each L1 core
var sprayResults = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput> [mapArity];
for (int i = 0; i < mapArity; i++)
{
sprayResults[i] = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput>(importLeft, a => this.mapper(a, null));
}
// [4] Union the shuffled data by group key
MultiUnionStreamable <TMapKey, TReduceInput> .l2index = 0;
var shuffleL2Result
= new MultiUnionStreamable <TMapKey, TReduceInput>(sprayResults, false) as MultiUnionStreamable <TMapKey, TOutput>;
return(shuffleL2Result.Subscribe(observer));
}
}
}
