private DryadQueryNode VisitSetOperation(QueryNodeInfo source1,
QueryNodeInfo source2,
QueryNodeType nodeType,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child1 = this.Visit(source1);
DryadQueryNode child2 = this.Visit(source2);
DryadQueryNode resNode = null;
Type keyType = child1.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
if (child1.IsDynamic || child2.IsDynamic)
{
// Well, let us do the simplest thing for now
child1 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
resNode = new DryadSetOperationNode(nodeType, nodeType.ToString(), comparerExpr,
queryExpr, child1, child2);
return resNode;
}
bool isDescending1 = child1.OutputDataSetInfo.orderByInfo.IsDescending;
bool isDescending2 = child2.OutputDataSetInfo.orderByInfo.IsDescending;
// Partition child1 and child2 if needed
if (child1.OutputPartition.ParType == PartitionType.Range &&
child1.OutputPartition.HasKeys &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Range ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Range distribute child2 using child1's partition
child2 = child1.OutputPartition.CreatePartitionNode(keySelectExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Range &&
child2.OutputPartition.HasKeys &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
// Range distribute child1 using child2's partition
child1 = child2.OutputPartition.CreatePartitionNode(keySelectExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else if (child1.OutputPartition.ParType == PartitionType.Hash &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Hash ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
!child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Hash distribute child2:
child2 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child1.OutputPartition.Count,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Hash &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
child1 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child2.OutputPartition.Count,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else
{
// No luck. Hash distribute both child1 and child2, then perform hash operation
int parCnt = Math.Max(child1.OutputPartition.Count, child2.OutputPartition.Count);
if (parCnt > 1)
{
child1 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
// Perform either hash or ordered operation
string opName = "";
if (child1.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort inner if unsorted
child2 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending1, queryExpr, child2);
}
opName = "Ordered";
}
else if (child2.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort outer if unsorted
child1 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending2, queryExpr, child1);
}
opName = "Ordered";
}
resNode = new DryadSetOperationNode(nodeType, opName + nodeType, comparerExpr, queryExpr, child1, child2);
return resNode;
}
internal DLinqSelectNode(QueryNodeType nodeType,
LambdaExpression selectExpr,
LambdaExpression resultSelectExpr,
Expression queryExpr,
DLinqQueryNode child)
: base(nodeType, child.QueryGen, queryExpr, child)
{
Debug.Assert(nodeType == QueryNodeType.Select || nodeType == QueryNodeType.SelectMany);
this.m_selectExpr = selectExpr;
this.m_resultSelectExpr = resultSelectExpr;
//If indexed version and the index is a long, we will use opName=DryadLong.
if (this.m_selectExpr.Parameters.Count() == 2 &&
this.m_selectExpr.Parameters[1].Type == typeof(long))
{
this.m_opName = "Long" + nodeType;
}
else
{
this.m_opName = nodeType.ToString();
}
this.m_partitionCount = child.OutputPartition.Count;
this.m_outputDataSetInfo = this.ComputeOutputDataSetInfo();
this.m_dynamicManager = this.InferDynamicManager();
}