public override IDisposable Subscribe(IStreamObserver <TMapKey, TOutput> observer) { // asymmetric mapper implies that we have to have a 2-input mapper Contract.Assert((!this.leftAsymmetric1) || (this.sourceRight1 != null)); Contract.Assert((!this.leftAsymmetric2) || (this.sourceRight2 != null)); var mapArity = Config.MapArity; var reduceArity = Config.ReduceArity; // process mapper #1 Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>[] shuffleL2Results1; if (this.sourceRight1 != null) // two-input mapper { // DEAD shuffleL2Results1 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> [reduceArity]; // [1] spray batches into L1 physical cores var importLeft1 = new SprayGroupImportStreamable <TMapKey, TMapInputLeft1>(this.sourceLeft1, mapArity, this.leftAsymmetric1); var importRight1 = new SprayGroupImportStreamable <TMapKey, TMapInputRight1>(this.sourceRight1, mapArity); // [2] perform the spray lambda on each L1 core var sprayResults1 = new BinaryMulticastStreamable <TMapKey, TMapInputLeft1, TMapInputRight1, TReduceInput1> [mapArity]; for (int i = 0; i < mapArity; i++) { sprayResults1[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft1, TMapInputRight1, TReduceInput1>(importLeft1, importRight1, this.mapper1); } // [3] apply shuffle on the result of each spray Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>[] shuffleL1Results1 = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey> [mapArity]; for (int i = 0; i < mapArity; i++) { shuffleL1Results1[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey>(sprayResults1[i], this.keySelector1, reduceArity, i); } // [4] Union the shuffled data by group key MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> .l2index = 0; for (int i = 0; i < reduceArity; i++) { shuffleL2Results1[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>(shuffleL1Results1); } } else // single-input mapper { // [1] spray batches into L1 physical cores var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft1> (this.sourceLeft1, mapArity, false, this.sprayComparer1); // [2] perform the spray lambda on each L1 core var sprayResults1 = new MulticastStreamable <TMapKey, TMapInputLeft1, TReduceInput1> [mapArity]; for (int i = 0; i < mapArity; i++) { sprayResults1[i] = new MulticastStreamable <TMapKey, TMapInputLeft1, TReduceInput1>(importLeft, a => this.mapper1(a, null)); } if (this.reduceInMap || (this.reduceOptions == OperationalHint.Asymmetric)) { shuffleL2Results1 = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> [mapArity]; for (int i = 0; i < mapArity; i++) { shuffleL2Results1[i] = new GroupNestedStreamable <TMapKey, TReduceInput1, TReduceKey>(sprayResults1[i], this.keySelector1); } } else { shuffleL2Results1 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> [reduceArity]; // [3] apply shuffle on the result of each spray Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>[] shuffleL1Results1 = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey> [mapArity]; for (int i = 0; i < mapArity; i++) { shuffleL1Results1[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey>(sprayResults1[i], this.keySelector1, reduceArity, i); } // [4] Union the shuffled data by group key MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> .l2index = 0; for (int i = 0; i < reduceArity; i++) { shuffleL2Results1[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>(shuffleL1Results1); } } } // process mapper #2 Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>[] shuffleL2Results2; if (this.sourceRight2 != null) // two-input mapper { // DEAD shuffleL2Results2 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> [reduceArity]; // [1] spray batches into L1 physical cores var importLeft2 = new SprayGroupImportStreamable <TMapKey, TMapInputLeft2>(this.sourceLeft2, mapArity, this.leftAsymmetric2); var importRight2 = new SprayGroupImportStreamable <TMapKey, TMapInputRight2>(this.sourceRight2, mapArity); // [2] perform the spray lambda on each L1 core var sprayResults2 = new BinaryMulticastStreamable <TMapKey, TMapInputLeft2, TMapInputRight2, TReduceInput2> [mapArity]; for (int i = 0; i < mapArity; i++) { sprayResults2[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft2, TMapInputRight2, TReduceInput2>(importLeft2, importRight2, this.mapper2); } // [3] apply shuffle on the result of each spray Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>[] shuffleL1Results2 = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey> [mapArity]; for (int i = 0; i < mapArity; i++) { shuffleL1Results2[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey>(sprayResults2[i], this.keySelector2, reduceArity, i); } // [4] Union the shuffled data by group key MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> .l2index = 0; for (int i = 0; i < reduceArity; i++) { shuffleL2Results2[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>(shuffleL1Results2); } } else // single-input mapper { // [1] spray batches into L1 physical cores var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft2>(this.sourceLeft2, mapArity, this.reduceOptions == OperationalHint.Asymmetric); // [2] perform the spray lambda on each L1 core var sprayResults2 = new MulticastStreamable <TMapKey, TMapInputLeft2, TReduceInput2> [mapArity]; for (int i = 0; i < mapArity; i++) { sprayResults2[i] = new MulticastStreamable <TMapKey, TMapInputLeft2, TReduceInput2>(importLeft, a => this.mapper2(a, null)); } if (this.reduceInMap || (this.reduceOptions == OperationalHint.Asymmetric)) { shuffleL2Results2 = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> [mapArity]; for (int i = 0; i < mapArity; i++) { shuffleL2Results2[i] = new GroupNestedStreamable <TMapKey, TReduceInput2, TReduceKey>(sprayResults2[i], this.keySelector2); } } else { shuffleL2Results2 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> [reduceArity]; // [3] apply shuffle on the result of each spray var shuffleL1Results2 = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey> [mapArity]; for (int i = 0; i < mapArity; i++) { shuffleL1Results2[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey>(sprayResults2[i], this.keySelector2, reduceArity, i); } // [4] Union the shuffled data by group key MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> .l2index = 0; for (int i = 0; i < reduceArity; i++) { shuffleL2Results2[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>(shuffleL1Results2); } } } // process 2-input reducer // [5] perform the apply lambda on each L2 core var innerResults = new BinaryMulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1, TReduceInput2, TBind> [shuffleL2Results1.Length]; var ungroupInnerResults = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput> [shuffleL2Results1.Length]; for (int i = 0; i < shuffleL2Results1.Length; i++) { innerResults[i] = new BinaryMulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1, TReduceInput2, TBind>(shuffleL2Results1[i], shuffleL2Results2[i], this.reducer); ungroupInnerResults[i] = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput>(this.sourceLeft1.Properties.KeyEqualityComparer, innerResults[i], this.resultSelector); } // [6] final single merging union var union = new MultiUnionStreamable <TMapKey, TOutput>(ungroupInnerResults, false); return(union.Subscribe(observer)); }
public override IDisposable Subscribe(IStreamObserver <TMapKey, TOutput> observer) { // asymmetric mapper implies that we have to have a 2-input mapper Contract.Assert((!this.leftAsymmetric) || (this.sourceRight != null)); var mapArity = this.isMulticore ? 1 : Config.MapArity; var reduceArity = this.isMulticore ? 1 : Config.ReduceArity; if (this.keySelector != null) { if (this.sourceRight != null) // two-input mapper { // [1] spray batches into L1 physical cores var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity, this.leftAsymmetric); var importRight = new SprayGroupImportStreamable <TMapKey, TMapInputRight>(this.sourceRight, mapArity); // [2] perform the spray lambda on each L1 core var sprayResults = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput> [mapArity]; for (int i = 0; i < mapArity; i++) { sprayResults[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput>(importLeft, importRight, this.mapper); } // [3] apply shuffle on the result of each spray Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>[] shuffleL1Results = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey> [mapArity]; for (int i = 0; i < mapArity; i++) { shuffleL1Results[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey>(this.keyComparer, sprayResults[i], this.keySelector, reduceArity, i); } // [4] Union the shuffled data by group key MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> .l2index = 0; var shuffleL2Results = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [reduceArity]; for (int i = 0; i < reduceArity; i++) { shuffleL2Results[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>(shuffleL1Results); } // [5] perform the apply lambda on each L2 core var innerResults = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind> [reduceArity]; var ungroupInnerResults = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput> [reduceArity]; for (int i = 0; i < reduceArity; i++) { innerResults[i] = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind>(shuffleL2Results[i], this.reducer); ungroupInnerResults[i] = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput>(this.sourceLeft.Properties.KeyEqualityComparer, innerResults[i], this.resultSelector); } // [6] final single merging union var union = new MultiUnionStreamable <TMapKey, TOutput>(ungroupInnerResults, false); return(union.Subscribe(observer)); } else // single-input mapper { // [1] spray batches into L1 physical cores var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity, this.leftAsymmetric, this.sprayComparer); // [2] perform the spray lambda on each L1 core var sprayResults = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput> [mapArity]; for (int i = 0; i < mapArity; i++) { sprayResults[i] = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput>(importLeft, a => this.mapper(a, null)); } Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>[] mergeInputs; if (this.reduceInMap) // apply reducer in map phase itself { // [3] apply shuffle on the result of each spray mergeInputs = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [mapArity]; for (int i = 0; i < mapArity; i++) { mergeInputs[i] = new GroupNestedStreamable <TMapKey, TReduceInput, TReduceKey>(this.keyComparer, sprayResults[i], this.keySelector); } } else { // [3] apply shuffle on the result of each spray Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>[] shuffleL1Results = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey> [mapArity]; for (int i = 0; i < mapArity; i++) { shuffleL1Results[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey>(this.keyComparer, sprayResults[i], this.keySelector, reduceArity, i); } // [4] Union the shuffled data by group key MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> .l2index = 0; mergeInputs = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [reduceArity]; mergeInputs = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [reduceArity]; for (int i = 0; i < reduceArity; i++) { mergeInputs[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>(shuffleL1Results); } } // [5] perform the apply lambda on each L2 core var innerResults = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind> [mergeInputs.Length]; var ungroupInnerResults = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput> [mergeInputs.Length]; for (int i = 0; i < mergeInputs.Length; i++) { innerResults[i] = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind>(mergeInputs[i], this.reducer); ungroupInnerResults[i] = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput>(this.sourceLeft.Properties.KeyEqualityComparer, innerResults[i], this.resultSelector); } // [6] final single merging union var union = new MultiUnionStreamable <TMapKey, TOutput>(ungroupInnerResults, false); return(union.Subscribe(observer)); } } else { if (this.sourceRight != null) // two-input mapper { // [1] spray batches into L1 physical cores var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity, this.leftAsymmetric); var importRight = new SprayGroupImportStreamable <TMapKey, TMapInputRight>(this.sourceRight, mapArity); // [2] perform the spray lambda on each L1 core var sprayResults = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput> [mapArity]; for (int i = 0; i < mapArity; i++) { sprayResults[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput>(importLeft, importRight, this.mapper); } // [4] Union the shuffled data by group key MultiUnionStreamable <TMapKey, TReduceInput> .l2index = 0; var shuffleL2Result = new MultiUnionStreamable <TMapKey, TReduceInput>(sprayResults, false) as MultiUnionStreamable <TMapKey, TOutput>; return(shuffleL2Result.Subscribe(observer)); } else // single-input mapper { // [1] spray batches into L1 physical cores var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity); // [2] perform the spray lambda on each L1 core var sprayResults = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput> [mapArity]; for (int i = 0; i < mapArity; i++) { sprayResults[i] = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput>(importLeft, a => this.mapper(a, null)); } // [4] Union the shuffled data by group key MultiUnionStreamable <TMapKey, TReduceInput> .l2index = 0; var shuffleL2Result = new MultiUnionStreamable <TMapKey, TReduceInput>(sprayResults, false) as MultiUnionStreamable <TMapKey, TOutput>; return(shuffleL2Result.Subscribe(observer)); } } }