private DLinqQueryNode VisitRangePartition(QueryNodeInfo source, LambdaExpression keySelectExpr, Expression keysExpr, Expression comparerExpr, Expression isDescendingExpr, Expression partitionCountExpr, Expression queryExpr) { DLinqQueryNode child = this.Visit(source); Type keyType = keySelectExpr.Type.GetGenericArguments()[1]; if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType)) { throw DryadLinqException.Create(DryadLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType), queryExpr); } DLinqQueryNode resNode; if (keysExpr == null) { // case: no keys are provided -- create range partitioner with auto-separator-selection bool dynamicOptEnabled = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicRangePartitionLevel) != 0; bool useDynamic = (dynamicOptEnabled || child.IsDynamic); // NOTE: for MayRTM, useDynamic should always be false resNode = this.CreateRangePartition(useDynamic, keySelectExpr, null, comparerExpr, isDescendingExpr, queryExpr, partitionCountExpr, child); } else { // case: keys are local enum (eg an array) -- create range partitioner with keys input via Object-store. resNode = new DLinqRangePartitionNode(keySelectExpr, null, keysExpr, comparerExpr, isDescendingExpr, null, queryExpr, child); resNode = new DLinqMergeNode(false, queryExpr, resNode); } return resNode; }
internal virtual string Visit(DLinqRangePartitionNode node, CodeMemberMethod vertexMethod, string[] readerNames, string[] writerNames) { return node.AddVertexCode(vertexMethod, readerNames, writerNames); }
// Creates an "auto-sampling range-partition sub-query" private DLinqQueryNode CreateRangePartition(bool isDynamic, LambdaExpression keySelectExpr, LambdaExpression resultSelectExpr, Expression comparerExpr, Expression isDescendingExpr, Expression queryExpr, Expression partitionCountExpr, DLinqQueryNode child) { // Make child a Tee node child.IsForked = true; // The partition count Expression countExpr = null; if (isDescendingExpr == null) { isDescendingExpr = Expression.Constant(false, typeof(bool)); //default for isDescending is false. } // NOTE: for MayRTM, isDynamic should never be true if (!isDynamic) { if (partitionCountExpr != null) { countExpr = partitionCountExpr; } else { // If partitionCount was not explicitly set, use the child's partition count. countExpr = Expression.Constant(child.OutputPartition.Count); } } Type recordType = child.OutputTypes[0]; Type keyType = keySelectExpr.Type.GetGenericArguments()[1]; // Create x => Phase1Sampling(x_1, keySelector, denv) Type lambdaParamType1 = typeof(IEnumerable<>).MakeGenericType(recordType); ParameterExpression lambdaParam1 = Expression.Parameter(lambdaParamType1, "x_1"); ParameterExpression denvParam = Expression.Parameter(typeof(VertexEnv), "denv"); MethodInfo minfo = typeof(DryadLinqSampler).GetMethod("Phase1Sampling"); Expression body = Expression.Call(minfo.MakeGenericMethod(recordType, keyType), lambdaParam1, keySelectExpr, denvParam); Type type = typeof(Func<,>).MakeGenericType(lambdaParam1.Type, body.Type); LambdaExpression samplingExpr = Expression.Lambda(type, body, lambdaParam1); // Create the Sampling node DLinqApplyNode samplingNode = new DLinqApplyNode(samplingExpr, queryExpr, child); // Create x => RangeSampler(x, keySelectExpr, comparer, isDescendingExpr) Type lambdaParamType = typeof(IEnumerable<>).MakeGenericType(keyType); ParameterExpression lambdaParam = Expression.Parameter(lambdaParamType, "x_2"); //For RTM, isDynamic should never be true. //string methodName = (isDynamic) ? "RangeSampler_Dynamic" : "RangeSampler_Static"; Debug.Assert(isDynamic == false, "Internal error: isDynamic is true."); string methodName = "RangeSampler_Static"; minfo = typeof(DryadLinqSampler).GetMethod(methodName); minfo = minfo.MakeGenericMethod(keyType); Expression comparerArgExpr = comparerExpr; if (comparerExpr == null) { if (!TypeSystem.HasDefaultComparer(keyType)) { throw DryadLinqException.Create(DryadLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType), queryExpr); } comparerArgExpr = Expression.Constant(null, typeof(IComparer<>).MakeGenericType(keyType)); } Expression lastArg; if (isDynamic) { lastArg = denvParam; } else { lastArg = countExpr; } body = Expression.Call(minfo, lambdaParam, comparerArgExpr, isDescendingExpr, lastArg); type = typeof(Func<,>).MakeGenericType(lambdaParam.Type, body.Type); LambdaExpression samplerExpr = Expression.Lambda(type, body, lambdaParam); // Create the sample node DLinqQueryNode sampleDataNode = new DLinqMergeNode(false, queryExpr, samplingNode); DLinqQueryNode sampleNode = new DLinqApplyNode(samplerExpr, queryExpr, sampleDataNode); sampleNode.IsForked = true; // Create the range distribute node DLinqQueryNode resNode = new DLinqRangePartitionNode(keySelectExpr, resultSelectExpr, null, comparerExpr, isDescendingExpr, countExpr, queryExpr, child, sampleNode); resNode = new DLinqMergeNode(false, queryExpr, resNode); // Set the dynamic manager for sampleNode if (isDynamic) { sampleDataNode.DynamicManager = new DynamicRangeDistributor(resNode); } return resNode; }