private DryadQueryNode VisitSetOperation(QueryNodeInfo source1,
QueryNodeInfo source2,
QueryNodeType nodeType,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child1 = this.Visit(source1);
DryadQueryNode child2 = this.Visit(source2);
DryadQueryNode resNode = null;
Type keyType = child1.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
if (child1.IsDynamic || child2.IsDynamic)
{
// Well, let us do the simplest thing for now
child1 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
resNode = new DryadSetOperationNode(nodeType, nodeType.ToString(), comparerExpr,
queryExpr, child1, child2);
return resNode;
}
bool isDescending1 = child1.OutputDataSetInfo.orderByInfo.IsDescending;
bool isDescending2 = child2.OutputDataSetInfo.orderByInfo.IsDescending;
// Partition child1 and child2 if needed
if (child1.OutputPartition.ParType == PartitionType.Range &&
child1.OutputPartition.HasKeys &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Range ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Range distribute child2 using child1's partition
child2 = child1.OutputPartition.CreatePartitionNode(keySelectExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Range &&
child2.OutputPartition.HasKeys &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
// Range distribute child1 using child2's partition
child1 = child2.OutputPartition.CreatePartitionNode(keySelectExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else if (child1.OutputPartition.ParType == PartitionType.Hash &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Hash ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
!child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Hash distribute child2:
child2 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child1.OutputPartition.Count,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Hash &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
child1 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child2.OutputPartition.Count,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else
{
// No luck. Hash distribute both child1 and child2, then perform hash operation
int parCnt = Math.Max(child1.OutputPartition.Count, child2.OutputPartition.Count);
if (parCnt > 1)
{
child1 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
// Perform either hash or ordered operation
string opName = "";
if (child1.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort inner if unsorted
child2 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending1, queryExpr, child2);
}
opName = "Ordered";
}
else if (child2.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort outer if unsorted
child1 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending2, queryExpr, child1);
}
opName = "Ordered";
}
resNode = new DryadSetOperationNode(nodeType, opName + nodeType, comparerExpr, queryExpr, child1, child2);
return resNode;
}
internal virtual string Visit(DryadSetOperationNode node,
CodeMemberMethod vertexMethod,
string[] readerNames,
string[] writerNames)
{
return node.AddVertexCode(vertexMethod, readerNames, writerNames);
}
