internal NodeSelectCriteria(QueryAxisType axis, NodeQName qname, QueryNodeType nodeType)
{
this.axis = QueryDataModel.GetAxis(axis);
this.qname = qname;
this.qnameType = qname.GetQNameType();
this.type = nodeType;
}
internal QueryAxis(QueryAxisType type, AxisDirection direction, QueryNodeType principalNode, QueryNodeType validNodeTypes)
{
this.direction = direction;
this.principalNode = principalNode;
this.type = type;
this.validNodeTypes = validNodeTypes;
}
internal QueryAxis(QueryAxisType type, AxisDirection direction, QueryNodeType principalNode, QueryNodeType validNodeTypes)
{
this.direction = direction;
this.principalNode = principalNode;
this.type = type;
this.validNodeTypes = validNodeTypes;
}
internal NodeSelectCriteria(QueryAxisType axis, NodeQName qname, QueryNodeType nodeType)
{
this.axis = QueryDataModel.GetAxis(axis);
this.qname = qname;
this.qnameType = qname.GetQNameType();
this.type = nodeType;
}
// Overload for grouping nodes (AND, OR)
public QueryNode(QueryNodeType type, QueryOp op, QueryNodeList childNodes)
{
this.Type = type;
this.Operator = op;
this.Column = "";
this.Value = JSONObj.nullJO;
this.Children = childNodes;
}
// Standard single value node
public QueryNode(QueryNodeType type, string column, QueryOp op, JSONObj value)
{
this.Type = type;
this.Column = column;
this.Operator = op;
this.Value = value;
this.Children = new QueryNodeList();
}
private QueryNode StartQueryNode(QueryNodeType type, ProcessingObjectType processAs)
{
var newNode = new QueryNode
{
Type = type,
};
processingObjectTypeStack.Push(processAs);
processingQueryNodeStack.Push(newNode);
return(newNode);
}
// Overload for BETWEEN query nodes
public QueryNode(QueryNodeType type, string column, QueryOp op, JSONObj value1, JSONObj value2)
{
// Between and In operators are the only ones that know to look at the values array
Debug.Assert(op == QueryOp.Btwn || op == QueryOp.In);
this.Type = type;
this.Column = column;
this.Operator = op;
this.Children = new QueryNodeList();
this.Value = JSONObj.obj;
Value.Add(value1);
Value.Add(value2);
}
public static QueryNode CreateRoot(QueryNodeType type, QueryRootType rootType,
params Action <QueryNode>[] actions)
{
var rootNode = new QueryNode
{
Type = type,
RootType = rootType,
};
actions?.ForEach(x => x.Invoke(rootNode));
return(rootNode);
}
private static void CombineAndOrNodes(QueryNodeType nodeType, List <IQueryNode> expressionNodes, List <QueryError> errors, NodesToStringPosition nodesToStringPosition)
{
var count = expressionNodes.Count;
if (count == 0)
{
return;
}
for (var i = count - 1; i >= 0; --i)
{
var currentNode = expressionNodes[i];
var nextNode = i < count - 1 ? expressionNodes[i + 1] : null;
var previousNode = i > 0 ? expressionNodes[i - 1] : null;
if (currentNode is CombinedNode combinedNode)
{
if (combinedNode.leaf)
{
var(startIndex, length) = nodesToStringPosition[currentNode];
if (currentNode.type == nodeType)
{
if (previousNode == null)
{
errors.Add(new QueryError(startIndex + length, $"Missing left-hand operand to combine with node {currentNode.type}."));
}
else
{
combinedNode.AddNode(previousNode);
expressionNodes.RemoveAt(i - 1);
// Update current index
--i;
}
if (nextNode == null)
{
errors.Add(new QueryError(startIndex + length, $"Missing right-hand operand to combine with node {currentNode.type}."));
}
else
{
combinedNode.AddNode(nextNode);
expressionNodes.RemoveAt(i + 1);
}
}
}
}
}
}
internal static XPathNodeType GetXPathNodeType(QueryNodeType type)
{
XPathNodeType nodeType = XPathNodeType.All;
switch (type)
{
default:
break;
case QueryNodeType.Attribute:
nodeType = XPathNodeType.Attribute;
break;
case QueryNodeType.Root:
nodeType = XPathNodeType.Root;
break;
case QueryNodeType.Namespace:
nodeType = XPathNodeType.Namespace;
break;
case QueryNodeType.Element:
nodeType = XPathNodeType.Element;
break;
case QueryNodeType.Comment:
nodeType = XPathNodeType.Comment;
break;
case QueryNodeType.Text:
nodeType = XPathNodeType.Text;
break;
case QueryNodeType.Processing:
nodeType = XPathNodeType.ProcessingInstruction;
break;
}
return(nodeType);
}
// Is it possible to select nodes matching the given criteria from nodes of the given type
internal static bool IsSelectPossible(QueryNodeType nodeType, NodeSelectCriteria desc)
{
if (0 != (nodeType & QueryNodeType.Attribute))
{
switch (desc.Axis.Type)
{
default:
return(false);
// Navigation is possible from attributes on these axes
case QueryAxisType.Self:
case QueryAxisType.Ancestor:
case QueryAxisType.AncestorOrSelf:
case QueryAxisType.Parent:
return(true);
}
}
else if (0 != (nodeType & QueryNodeType.Root))
{
if (AxisDirection.Reverse == desc.Axis.Direction)
{
return(false);
}
switch (desc.Axis.Type)
{
default:
return(true);
// Navigation is possible from attributes on these axes
case QueryAxisType.Attribute:
case QueryAxisType.Namespace:
return(false);
}
}
return(true);
}
public NullaryQueryExpr(QueryNodeType nodeType)
: base(nodeType)
{
}
public UnaryQueryExpr(QueryNodeType nodeType, QueryExpr source, params QueryExpr[] arguments)
: base(nodeType, new[] { source }.Concat(arguments).ToArray())
{
Source = source;
}
public EnumerableCreatorAttribute(QueryNodeType nodeType)
{
this.nodeType = nodeType;
}
internal static NodeSelectCriteria Create(QueryAxisType axis, NodeQName qname, QueryNodeType nodeType)
{
return new NodeSelectCriteria(axis, qname, nodeType);
}
/// <summary>
/// Initializes a new instance of an object.
/// </summary>
/// <param name="type">Query node type.</param>
protected QueryNode(QueryNodeType type)
{
this.Type = type;
}
private DryadQueryNode VisitSetOperation(QueryNodeInfo source1,
QueryNodeInfo source2,
QueryNodeType nodeType,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode child1 = this.Visit(source1);
DryadQueryNode child2 = this.Visit(source2);
DryadQueryNode resNode = null;
Type keyType = child1.OutputTypes[0];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
if (child1.IsDynamic || child2.IsDynamic)
{
// Well, let us do the simplest thing for now
child1 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr,
null,
StaticConfig.DefaultPartitionCount,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
resNode = new DryadSetOperationNode(nodeType, nodeType.ToString(), comparerExpr,
queryExpr, child1, child2);
return resNode;
}
bool isDescending1 = child1.OutputDataSetInfo.orderByInfo.IsDescending;
bool isDescending2 = child2.OutputDataSetInfo.orderByInfo.IsDescending;
// Partition child1 and child2 if needed
if (child1.OutputPartition.ParType == PartitionType.Range &&
child1.OutputPartition.HasKeys &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Range ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Range distribute child2 using child1's partition
child2 = child1.OutputPartition.CreatePartitionNode(keySelectExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Range &&
child2.OutputPartition.HasKeys &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
// Range distribute child1 using child2's partition
child1 = child2.OutputPartition.CreatePartitionNode(keySelectExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else if (child1.OutputPartition.ParType == PartitionType.Hash &&
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
if (child2.OutputPartition.ParType != PartitionType.Hash ||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
!child1.OutputPartition.IsSamePartition(child2.OutputPartition))
{
// Hash distribute child2:
child2 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child1.OutputPartition.Count,
queryExpr,
child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
else if (child2.OutputPartition.ParType == PartitionType.Hash &&
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
{
child1 = new DryadHashPartitionNode(keySelectExpr,
comparerExpr,
child2.OutputPartition.Count,
queryExpr,
child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
}
else
{
// No luck. Hash distribute both child1 and child2, then perform hash operation
int parCnt = Math.Max(child1.OutputPartition.Count, child2.OutputPartition.Count);
if (parCnt > 1)
{
child1 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child1);
if (IsMergeNodeNeeded(child1))
{
child1 = new DryadMergeNode(false, false, queryExpr, child1);
}
child2 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child2);
if (IsMergeNodeNeeded(child2))
{
child2 = new DryadMergeNode(false, false, queryExpr, child2);
}
}
}
// Perform either hash or ordered operation
string opName = "";
if (child1.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort inner if unsorted
child2 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending1, queryExpr, child2);
}
opName = "Ordered";
}
else if (child2.IsOrderedBy(keySelectExpr, comparer))
{
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
isDescending1 != isDescending2)
{
// Sort outer if unsorted
child1 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending2, queryExpr, child1);
}
opName = "Ordered";
}
resNode = new DryadSetOperationNode(nodeType, opName + nodeType, comparerExpr, queryExpr, child1, child2);
return resNode;
}
private DryadQueryNode VisitPartitionOp(string opName,
QueryNodeInfo source,
QueryNodeType nodeType,
Expression controlExpr,
MethodCallExpression queryExpr)
{
DryadQueryNode resNode;
if (nodeType == QueryNodeType.TakeWhile &&
controlExpr.Type.GetGenericArguments().Length != 2)
{
// The "indexed" version.
resNode = this.Visit(source);
// The following block used to be skipped for resNode.OutputPartition.Count == 1,
// which causes compilation error (bug 13593)
// @@TODO[p3] : implement a working optimization for nPartition==1 that calls
// directly to Linq TakeWhile.
// Note: the test is: if (resNode.IsDynamic || resNode.OutputPartition.Count > 1)
{
resNode.IsForked = true;
bool isLong = (queryExpr.Method.Name == "LongTakeWhile");
DryadQueryNode offsetNode = this.CreateOffset(isLong, queryExpr, resNode);
// Create (x, y) => GroupIndexedTakeWhile(x, y, controlExpr)
Type ptype1 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]);
Type ptype2 = typeof(IEnumerable<>).MakeGenericType(typeof(IndexedValue<long>));
ParameterExpression param1 = Expression.Parameter(ptype1, HpcLinqCodeGen.MakeUniqueName("x"));
ParameterExpression param2 = Expression.Parameter(ptype2, HpcLinqCodeGen.MakeUniqueName("y"));
string methodName = "GroupIndexed" + queryExpr.Method.Name;
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod(methodName);
minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]);
Expression body = Expression.Call(minfo, param1, param2, controlExpr);
Type type = typeof(Func<,,>).MakeGenericType(ptype1, ptype2, body.Type);
LambdaExpression procFunc = Expression.Lambda(type, body, param1, param2);
resNode = new DryadApplyNode(procFunc, queryExpr, resNode, offsetNode);
}
}
else if (!source.IsForked &&
(nodeType == QueryNodeType.Take || nodeType == QueryNodeType.TakeWhile) &&
(source.OperatorName == "OrderBy" || source.OperatorName == "OrderByDescending"))
{
resNode = this.Visit(source.children[0].child);
bool isDescending = (source.OperatorName == "OrderByDescending");
MethodCallExpression sourceQueryExpr = (MethodCallExpression)source.queryExpression;
LambdaExpression keySelectExpr = HpcLinqExpression.GetLambda(sourceQueryExpr.Arguments[1]);
Expression comparerExpr = null;
if (sourceQueryExpr.Arguments.Count == 3)
{
comparerExpr = sourceQueryExpr.Arguments[2];
}
resNode = this.PromoteConcat(
source.children[0].child,
resNode,
delegate(DryadQueryNode x) {
DryadQueryNode y = new DryadOrderByNode(keySelectExpr, comparerExpr,
isDescending, sourceQueryExpr, x);
return FirstStagePartitionOp(opName, nodeType, controlExpr, queryExpr, y);
});
if (resNode.IsDynamic || resNode.OutputPartition.Count > 1)
{
// Need a mergesort
resNode = new DryadMergeNode(keySelectExpr, comparerExpr, isDescending, false, sourceQueryExpr, resNode);
}
}
else
{
resNode = this.Visit(source);
if (nodeType == QueryNodeType.Take || nodeType == QueryNodeType.TakeWhile)
{
resNode = this.PromoteConcat(
source, resNode,
x => FirstStagePartitionOp(opName, nodeType, controlExpr, queryExpr, x));
}
}
resNode = new DryadPartitionOpNode(opName, nodeType, controlExpr, false, queryExpr, resNode);
return resNode;
}
internal static XPathNodeType GetXPathNodeType(QueryNodeType type)
{
XPathNodeType nodeType = XPathNodeType.All;
switch(type)
{
default:
break;
case QueryNodeType.Attribute:
nodeType = XPathNodeType.Attribute;
break;
case QueryNodeType.Root:
nodeType = XPathNodeType.Root;
break;
case QueryNodeType.Namespace:
nodeType = XPathNodeType.Namespace;
break;
case QueryNodeType.Element:
nodeType = XPathNodeType.Element;
break;
case QueryNodeType.Comment:
nodeType = XPathNodeType.Comment;
break;
case QueryNodeType.Text:
nodeType = XPathNodeType.Text;
break;
case QueryNodeType.Processing:
nodeType = XPathNodeType.ProcessingInstruction;
break;
}
return nodeType;
}
internal static NodeSelectCriteria Create(QueryAxisType axis, NodeQName qname, QueryNodeType nodeType)
{
return(new NodeSelectCriteria(axis, qname, nodeType));
}
internal DLinqPartitionOpNode(string opName,
QueryNodeType nodeType,
Expression controlExpr,
bool isFirstStage,
Expression queryExpr,
DLinqQueryNode child)
: base(nodeType, child.QueryGen, queryExpr, child)
{
this.m_controlExpression = controlExpr;
this.m_isFirstStage = isFirstStage;
this.m_opName = opName;
this.m_count = -1;
if (nodeType == QueryNodeType.Take || nodeType == QueryNodeType.Skip)
{
ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
this.m_count = evaluator.Eval(controlExpr);
}
DataSetInfo childInfo = child.OutputDataSetInfo;
if (isFirstStage)
{
this.m_partitionCount = child.OutputPartition.Count;
this.m_outputDataSetInfo = new DataSetInfo(child.OutputDataSetInfo);
this.m_dynamicManager = this.InferDynamicManager();
}
else
{
this.m_partitionCount = 1;
this.m_outputDataSetInfo = new DataSetInfo(childInfo);
this.m_outputDataSetInfo.partitionInfo = DataSetInfo.OnePartition;
if (childInfo.partitionInfo.Count > 1 &&
(childInfo.partitionInfo.ParType != PartitionType.Range ||
childInfo.orderByInfo.IsOrdered))
{
this.m_outputDataSetInfo.orderByInfo = DataSetInfo.NoOrderBy;
}
this.m_dynamicManager = DynamicManager.None;
}
}
internal DLinqJoinNode(QueryNodeType nodeType,
string opName,
LambdaExpression outerKeySelectExpr,
LambdaExpression innerKeySelectExpr,
LambdaExpression resultSelectExpr,
Expression comparerExpr,
Expression queryExpr,
DLinqQueryNode outerChild,
DLinqQueryNode innerChild)
: base(nodeType, outerChild.QueryGen, queryExpr, outerChild, innerChild)
{
Debug.Assert(nodeType == QueryNodeType.Join || nodeType == QueryNodeType.GroupJoin);
this.m_outerKeySelectExpr = outerKeySelectExpr;
this.m_innerKeySelectExpr = innerKeySelectExpr;
this.m_resultSelectExpr = resultSelectExpr;
this.m_comparerExpr = comparerExpr;
this.m_opName = opName;
this.m_attachedPipeline = null;
this.m_comparer = null;
this.m_comparerIdx = -1;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
this.m_comparer = evaluator.Eval(comparerExpr);
this.m_comparerIdx = DryadLinqObjectStore.Put(m_comparer);
}
if (StaticConfig.UseMemoryFIFO)
{
bool isStateful = this.IsStateful;
foreach (DLinqQueryNode child in this.Children)
{
if (!(child is DLinqInputNode) &&
!child.IsForked &&
!(isStateful && child.IsStateful) &&
child.PartitionCount > 1)
{
child.ChannelType = ChannelType.MemoryFIFO;
}
isStateful = isStateful || child.IsStateful;
}
}
this.m_partitionCount = outerChild.OutputDataSetInfo.partitionInfo.Count;
this.m_outputDataSetInfo = this.ComputeOutputDataSetInfo();
this.m_dynamicManager = DynamicManager.None;
}
internal DLinqSetOperationNode(QueryNodeType nodeType,
string opName,
Expression comparerExpr,
Expression queryExpr,
DLinqQueryNode child1,
DLinqQueryNode child2)
: base(nodeType, child1.QueryGen, queryExpr, child1, child2)
{
this.m_isOrdered = opName.StartsWith("Ordered", StringComparison.Ordinal);
this.m_opName = opName;
this.m_comparerExpr = comparerExpr;
this.m_comparer = null;
this.m_comparerIdx = -1;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
this.m_comparer = evaluator.Eval(comparerExpr);
this.m_comparerIdx = DryadLinqObjectStore.Put(m_comparer);
}
if (StaticConfig.UseMemoryFIFO)
{
bool isStateful = this.IsStateful;
foreach (DLinqQueryNode child in this.Children)
{
if (!(child is DLinqInputNode) &&
!child.IsForked &&
!(isStateful && child.IsStateful) &&
child.PartitionCount > 1)
{
child.ChannelType = ChannelType.MemoryFIFO;
}
isStateful = isStateful || child.IsStateful;
}
}
this.m_partitionCount = child1.OutputDataSetInfo.partitionInfo.Count;
this.m_outputDataSetInfo = new DataSetInfo(child1.OutputDataSetInfo);
this.m_dynamicManager = DynamicManager.None;
}
private DryadQueryNode FirstStagePartitionOp(string opName,
QueryNodeType nodeType,
Expression controlExpr,
MethodCallExpression queryExpr,
DryadQueryNode child)
{
if (nodeType == QueryNodeType.TakeWhile)
{
Type ptype = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
ParameterExpression param = Expression.Parameter(ptype, HpcLinqCodeGen.MakeUniqueName("x"));
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod("GroupTakeWhile");
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
Expression body = Expression.Call(minfo, param, controlExpr);
Type type = typeof(Func<,>).MakeGenericType(ptype, body.Type);
LambdaExpression procFunc = Expression.Lambda(type, body, param);
return new DryadApplyNode(procFunc, queryExpr, child);
}
else
{
return new DryadPartitionOpNode(opName, nodeType, controlExpr, true, queryExpr, child);
}
}
// Is it possible to select nodes matching the given criteria from nodes of the given type
internal static bool IsSelectPossible(QueryNodeType nodeType, NodeSelectCriteria desc)
{
if (0 != (nodeType & QueryNodeType.Attribute))
{
switch(desc.Axis.Type)
{
default:
return false;
// Navigation is possible from attributes on these axes
case QueryAxisType.Self:
case QueryAxisType.Ancestor:
case QueryAxisType.AncestorOrSelf:
case QueryAxisType.Parent:
return true;
}
}
else if (0 != (nodeType & QueryNodeType.Root))
{
if (AxisDirection.Reverse == desc.Axis.Direction)
{
return false;
}
switch(desc.Axis.Type)
{
default:
return true;
// Navigation is possible from attributes on these axes
case QueryAxisType.Attribute:
case QueryAxisType.Namespace:
return false;
}
}
return true;
}
private DryadQueryNode VisitJoin(QueryNodeInfo outerSource,
QueryNodeInfo innerSource,
QueryNodeType nodeType,
LambdaExpression outerKeySelector,
LambdaExpression innerKeySelector,
LambdaExpression resultSelector,
Expression comparerExpr,
Expression queryExpr)
{
DryadQueryNode outerChild = this.Visit(outerSource);
DryadQueryNode innerChild = this.Visit(innerSource);
DryadQueryNode joinNode = null;
Type keyType = outerKeySelector.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
queryExpr);
}
// The comparer object:
object comparer = null;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
comparer = evaluator.Eval(comparerExpr);
}
if (outerChild.IsDynamic || innerChild.IsDynamic)
{
// Well, let us do the simplest thing for now
outerChild = new DryadHashPartitionNode(outerKeySelector,
comparerExpr,
StaticConfig.DefaultPartitionCount,
queryExpr,
outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
innerChild = new DryadHashPartitionNode(innerKeySelector,
comparerExpr,
StaticConfig.DefaultPartitionCount,
queryExpr,
innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
joinNode = new DryadJoinNode(nodeType,
"Hash" + nodeType,
outerKeySelector,
innerKeySelector,
resultSelector,
comparerExpr,
queryExpr,
outerChild,
innerChild);
return joinNode;
}
bool isOuterDescending = outerChild.OutputDataSetInfo.orderByInfo.IsDescending;
bool isInnerDescending = innerChild.OutputDataSetInfo.orderByInfo.IsDescending;
// Partition outer and inner if needed
if (outerChild.OutputPartition.ParType == PartitionType.Range &&
outerChild.OutputPartition.HasKeys &&
outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
{
if (innerChild.OutputPartition.ParType != PartitionType.Range ||
!innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
!outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
{
// Range distribute inner using outer's partition.
innerChild = outerChild.OutputPartition.CreatePartitionNode(innerKeySelector, innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
}
}
else if (innerChild.OutputPartition.ParType == PartitionType.Range &&
innerChild.OutputPartition.HasKeys &&
innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
{
// Range distribute outer using inner's partition.
outerChild = innerChild.OutputPartition.CreatePartitionNode(outerKeySelector, outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
}
else if (outerChild.OutputPartition.ParType == PartitionType.Hash &&
outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
{
if (innerChild.OutputPartition.ParType != PartitionType.Hash ||
!innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
!outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
{
innerChild = new DryadHashPartitionNode(innerKeySelector,
comparerExpr,
outerChild.OutputPartition.Count,
queryExpr,
innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
}
}
else if (innerChild.OutputPartition.ParType == PartitionType.Hash &&
innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
{
outerChild = new DryadHashPartitionNode(outerKeySelector,
comparerExpr,
innerChild.OutputPartition.Count,
queryExpr,
outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
}
else
{
// No luck. Hash partition both outer and inner
int parCnt = Math.Max(outerChild.OutputPartition.Count, innerChild.OutputPartition.Count);
if (parCnt > 1)
{
outerChild = new DryadHashPartitionNode(outerKeySelector,
comparerExpr,
parCnt,
queryExpr,
outerChild);
if (IsMergeNodeNeeded(outerChild))
{
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
}
innerChild = new DryadHashPartitionNode(innerKeySelector,
comparerExpr,
parCnt,
queryExpr,
innerChild);
if (IsMergeNodeNeeded(innerChild))
{
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
}
}
}
// Perform either merge or hash join
string opName = "Hash";
if (outerChild.IsOrderedBy(outerKeySelector, comparer))
{
if (!innerChild.IsOrderedBy(innerKeySelector, comparer) ||
isOuterDescending != isInnerDescending)
{
// Sort inner if unsorted
innerChild = new DryadOrderByNode(innerKeySelector, comparerExpr,
isOuterDescending, queryExpr, innerChild);
}
opName = "Merge";
}
else if (innerChild.IsOrderedBy(innerKeySelector, comparer))
{
if (!outerChild.IsOrderedBy(outerKeySelector, comparer) ||
isOuterDescending != isInnerDescending)
{
// Sort outer if unsorted
outerChild = new DryadOrderByNode(outerKeySelector, comparerExpr,
isInnerDescending, queryExpr, outerChild);
}
opName = "Merge";
}
joinNode = new DryadJoinNode(nodeType,
opName + nodeType,
outerKeySelector,
innerKeySelector,
resultSelector,
comparerExpr,
queryExpr,
outerChild,
innerChild);
return joinNode;
}
internal DLinqQueryNode(QueryNodeType nodeType,
DryadLinqQueryGen queryGen,
Expression queryExpr,
params DLinqQueryNode[] children)
{
this.m_nodeType = nodeType;
this.m_queryGen = queryGen;
this.m_queryExpression = queryExpr;
this.m_parents = new List<DLinqQueryNode>(1);
this.m_children = children;
foreach (DLinqQueryNode child in children)
{
child.Parents.Add(this);
}
this.m_superNode = null;
this.m_isForked = false;
this.m_uniqueId = DryadLinqQueryGen.StartPhaseId;
this.m_channelType = ChannelType.DiskFile;
this.m_conOpType = ConnectionOpType.Pointwise;
this.m_opName = null;
this.m_vertexEntryMethod = null;
this.m_outputDataSetInfo = null;
this.m_partitionCount = -1;
this.m_dynamicManager = null;
}
private DryadQueryNode VisitSelect(QueryNodeInfo source,
QueryNodeType nodeType,
LambdaExpression selector,
LambdaExpression resultSelector,
MethodCallExpression queryExpr)
{
DryadQueryNode selectNode;
if (selector.Type.GetGenericArguments().Length == 2)
{
// If this select's child is a groupby node, push this select into its child, if
// 1. The groupby node is not tee'd, and
// 2. The groupby node has no result selector, and
// 3. The selector is decomposable
if (!source.IsForked &&
IsGroupByWithoutResultSelector(source.queryExpression) &&
Decomposition.GetDecompositionInfoList(selector, m_codeGen) != null)
{
MethodCallExpression expr = (MethodCallExpression)source.queryExpression;
LambdaExpression keySelectExpr = HpcLinqExpression.GetLambda(expr.Arguments[1]);
// Figure out elemSelectExpr and comparerExpr
LambdaExpression elemSelectExpr = null;
Expression comparerExpr = null;
if (expr.Arguments.Count == 3)
{
elemSelectExpr = HpcLinqExpression.GetLambda(expr.Arguments[2]);
if (elemSelectExpr == null)
{
comparerExpr = expr.Arguments[2];
}
}
else if (expr.Arguments.Count == 4)
{
elemSelectExpr = HpcLinqExpression.GetLambda(expr.Arguments[2]);
comparerExpr = expr.Arguments[3];
}
// Construct new query expression by building result selector expression
// and pushing it to groupby node.
selectNode = VisitGroupBy(source.children[0].child, keySelectExpr,
elemSelectExpr, selector, comparerExpr, queryExpr);
if (nodeType == QueryNodeType.SelectMany)
{
Type selectorRetType = selector.Type.GetGenericArguments()[1];
LambdaExpression id = IdentityFunction.Instance(selectorRetType);
selectNode = new DryadSelectNode(nodeType, id, resultSelector, queryExpr, selectNode);
}
}
else
{
DryadQueryNode child = this.Visit(source);
selectNode = this.PromoteConcat(
source, child,
x => new DryadSelectNode(nodeType, selector, resultSelector, queryExpr, x));
}
}
else
{
// The "indexed" version
DryadQueryNode child = this.Visit(source);
if (!child.IsDynamic && child.OutputPartition.Count == 1)
{
selectNode = this.PromoteConcat(
source, child,
x => new DryadSelectNode(nodeType, selector, resultSelector, queryExpr, x));
}
else
{
child.IsForked = true;
// Create (x, y) => Select(x, y, selector)
Type ptype1 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
Type ptype2 = typeof(IEnumerable<>).MakeGenericType(typeof(IndexedValue<long>));
ParameterExpression param1 = Expression.Parameter(ptype1, HpcLinqCodeGen.MakeUniqueName("x"));
ParameterExpression param2 = Expression.Parameter(ptype2, HpcLinqCodeGen.MakeUniqueName("y"));
string methodName = queryExpr.Method.Name;
Type[] selectorTypeArgs = selector.Type.GetGenericArguments();
Type typeArg2 = selectorTypeArgs[selectorTypeArgs.Length - 1];
if (nodeType == QueryNodeType.SelectMany)
{
if (resultSelector != null)
{
methodName += "Result";
}
typeArg2 = typeArg2.GetGenericArguments()[0];
}
string targetMethodName = methodName + "WithStartIndex";
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod(targetMethodName);
Expression body;
if (resultSelector == null)
{
minfo = minfo.MakeGenericMethod(child.OutputTypes[0], typeArg2);
body = Expression.Call(minfo, param1, param2, selector);
}
else
{
minfo = minfo.MakeGenericMethod(child.OutputTypes[0], typeArg2, resultSelector.Body.Type);
body = Expression.Call(minfo, param1, param2, selector, resultSelector);
}
Type type = typeof(Func<,,>).MakeGenericType(ptype1, ptype2, body.Type);
LambdaExpression procFunc = Expression.Lambda(type, body, param1, param2);
bool isLong = methodName.StartsWith("Long", StringComparison.Ordinal);
DryadQueryNode offsetNode = this.CreateOffset(isLong, queryExpr, child);
selectNode = new DryadApplyNode(procFunc, queryExpr, child, offsetNode);
}
}
return selectNode;
}
internal DLinqSelectNode(QueryNodeType nodeType,
LambdaExpression selectExpr,
LambdaExpression resultSelectExpr,
Expression queryExpr,
DLinqQueryNode child)
: base(nodeType, child.QueryGen, queryExpr, child)
{
Debug.Assert(nodeType == QueryNodeType.Select || nodeType == QueryNodeType.SelectMany);
this.m_selectExpr = selectExpr;
this.m_resultSelectExpr = resultSelectExpr;
//If indexed version and the index is a long, we will use opName=DryadLong.
if (this.m_selectExpr.Parameters.Count() == 2 &&
this.m_selectExpr.Parameters[1].Type == typeof(long))
{
this.m_opName = "Long" + nodeType;
}
else
{
this.m_opName = nodeType.ToString();
}
this.m_partitionCount = child.OutputPartition.Count;
this.m_outputDataSetInfo = this.ComputeOutputDataSetInfo();
this.m_dynamicManager = this.InferDynamicManager();
}
NodeSelectCriteria ParseNodeTest(QueryAxisType axisType)
{
Fx.Assert(QueryAxisType.None != axisType, "");
QueryAxis axis = QueryDataModel.GetAxis(axisType);
XPathToken token;
NodeQName qname = NodeQName.Empty;
if (null != (token = this.NextTokenClass(XPathTokenID.NameTest)))
{
switch (token.TokenID)
{
default:
this.ThrowError(QueryCompileError.UnexpectedToken);
break;
case XPathTokenID.Wildcard:
qname = new NodeQName(QueryDataModel.Wildcard, QueryDataModel.Wildcard);
break;
case XPathTokenID.NameTest:
qname = this.QualifyName(token.Prefix, token.Name);
break;
case XPathTokenID.NameWildcard:
qname = this.QualifyName(token.Prefix, QueryDataModel.Wildcard);
break;
}
}
QueryNodeType nodeType = QueryNodeType.Any;
if (qname.IsEmpty)
{
// Check for nodeTests
if (null == (token = this.NextTokenClass(XPathTokenID.NodeType)))
{
// Not a NodeTest either.
return(null);
}
switch (token.TokenID)
{
default:
this.ThrowError(QueryCompileError.UnsupportedNodeTest);
break;
case XPathTokenID.Comment:
nodeType = QueryNodeType.Comment;
break;
case XPathTokenID.Text:
nodeType = QueryNodeType.Text;
break;
case XPathTokenID.Processing:
nodeType = QueryNodeType.Processing;
break;
case XPathTokenID.Node:
nodeType = QueryNodeType.All;
break;
}
// Make sure the nodes being selected CAN actually be selected from this axis
if (0 == (axis.ValidNodeTypes & nodeType))
{
this.ThrowError(QueryCompileError.InvalidNodeType);
}
// Eat ()
this.NextToken(XPathTokenID.LParen, QueryCompileError.InvalidNodeTest);
this.NextToken(XPathTokenID.RParen, QueryCompileError.InvalidNodeTest);
}
else
{
nodeType = axis.PrincipalNodeType;
}
return(new NodeSelectCriteria(axisType, qname, nodeType));
}
public QueryNode(QueryNodeType nodeType, Expression condition)
{
this.NodeType = nodeType;
this.Condition = condition;
}
