private DryadQueryNode VisitJoin(QueryNodeInfo outerSource, QueryNodeInfo innerSource, QueryNodeType nodeType, LambdaExpression outerKeySelector, LambdaExpression innerKeySelector, LambdaExpression resultSelector, Expression comparerExpr, Expression queryExpr) { DryadQueryNode outerChild = this.Visit(outerSource); DryadQueryNode innerChild = this.Visit(innerSource); DryadQueryNode joinNode = null; Type keyType = outerKeySelector.Type.GetGenericArguments()[1]; if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType)) { throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType), queryExpr); } // The comparer object: object comparer = null; if (comparerExpr != null) { ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>(); comparer = evaluator.Eval(comparerExpr); } if (outerChild.IsDynamic || innerChild.IsDynamic) { // Well, let us do the simplest thing for now outerChild = new DryadHashPartitionNode(outerKeySelector, comparerExpr, StaticConfig.DefaultPartitionCount, queryExpr, outerChild); if (IsMergeNodeNeeded(outerChild)) { outerChild = new DryadMergeNode(false, false, queryExpr, outerChild); } innerChild = new DryadHashPartitionNode(innerKeySelector, comparerExpr, StaticConfig.DefaultPartitionCount, queryExpr, innerChild); if (IsMergeNodeNeeded(innerChild)) { innerChild = new DryadMergeNode(false, false, queryExpr, innerChild); } joinNode = new DryadJoinNode(nodeType, "Hash" + nodeType, outerKeySelector, innerKeySelector, resultSelector, comparerExpr, queryExpr, outerChild, innerChild); return joinNode; } bool isOuterDescending = outerChild.OutputDataSetInfo.orderByInfo.IsDescending; bool isInnerDescending = innerChild.OutputDataSetInfo.orderByInfo.IsDescending; // Partition outer and inner if needed if (outerChild.OutputPartition.ParType == PartitionType.Range && outerChild.OutputPartition.HasKeys && outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer)) { if (innerChild.OutputPartition.ParType != PartitionType.Range || !innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) || !outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition)) { // Range distribute inner using outer's partition. innerChild = outerChild.OutputPartition.CreatePartitionNode(innerKeySelector, innerChild); if (IsMergeNodeNeeded(innerChild)) { innerChild = new DryadMergeNode(false, false, queryExpr, innerChild); } } } else if (innerChild.OutputPartition.ParType == PartitionType.Range && innerChild.OutputPartition.HasKeys && innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer)) { // Range distribute outer using inner's partition. outerChild = innerChild.OutputPartition.CreatePartitionNode(outerKeySelector, outerChild); if (IsMergeNodeNeeded(outerChild)) { outerChild = new DryadMergeNode(false, false, queryExpr, outerChild); } } else if (outerChild.OutputPartition.ParType == PartitionType.Hash && outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer)) { if (innerChild.OutputPartition.ParType != PartitionType.Hash || !innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) || !outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition)) { innerChild = new DryadHashPartitionNode(innerKeySelector, comparerExpr, outerChild.OutputPartition.Count, queryExpr, innerChild); if (IsMergeNodeNeeded(innerChild)) { innerChild = new DryadMergeNode(false, false, queryExpr, innerChild); } } } else if (innerChild.OutputPartition.ParType == PartitionType.Hash && innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer)) { outerChild = new DryadHashPartitionNode(outerKeySelector, comparerExpr, innerChild.OutputPartition.Count, queryExpr, outerChild); if (IsMergeNodeNeeded(outerChild)) { outerChild = new DryadMergeNode(false, false, queryExpr, outerChild); } } else { // No luck. Hash partition both outer and inner int parCnt = Math.Max(outerChild.OutputPartition.Count, innerChild.OutputPartition.Count); if (parCnt > 1) { outerChild = new DryadHashPartitionNode(outerKeySelector, comparerExpr, parCnt, queryExpr, outerChild); if (IsMergeNodeNeeded(outerChild)) { outerChild = new DryadMergeNode(false, false, queryExpr, outerChild); } innerChild = new DryadHashPartitionNode(innerKeySelector, comparerExpr, parCnt, queryExpr, innerChild); if (IsMergeNodeNeeded(innerChild)) { innerChild = new DryadMergeNode(false, false, queryExpr, innerChild); } } } // Perform either merge or hash join string opName = "Hash"; if (outerChild.IsOrderedBy(outerKeySelector, comparer)) { if (!innerChild.IsOrderedBy(innerKeySelector, comparer) || isOuterDescending != isInnerDescending) { // Sort inner if unsorted innerChild = new DryadOrderByNode(innerKeySelector, comparerExpr, isOuterDescending, queryExpr, innerChild); } opName = "Merge"; } else if (innerChild.IsOrderedBy(innerKeySelector, comparer)) { if (!outerChild.IsOrderedBy(outerKeySelector, comparer) || isOuterDescending != isInnerDescending) { // Sort outer if unsorted outerChild = new DryadOrderByNode(outerKeySelector, comparerExpr, isInnerDescending, queryExpr, outerChild); } opName = "Merge"; } joinNode = new DryadJoinNode(nodeType, opName + nodeType, outerKeySelector, innerKeySelector, resultSelector, comparerExpr, queryExpr, outerChild, innerChild); return joinNode; }
internal virtual string Visit(DryadJoinNode node, CodeMemberMethod vertexMethod, string[] readerNames, string[] writerNames) { return node.AddVertexCode(vertexMethod, readerNames, writerNames); }