private DryadQueryNode VisitMultiApply(QueryNodeInfo source,
LambdaExpression procLambda,
bool perPartition,
bool isFirstOnly,
MethodCallExpression queryExpr)
{
DryadQueryNode[] childs = new DryadQueryNode[source.children.Count];
for (int i = 0; i < source.children.Count; ++i)
{
childs[i] = this.Visit(source.children[i].child);
}
bool isDynamic = childs.Any(x => x.IsDynamic);
if (perPartition && !isDynamic)
{
// Homomorphic case.
if (isFirstOnly)
{
for (int i = 1; i < childs.Length; ++i)
{
childs[i] = new DryadTeeNode(childs[i].OutputTypes[0], true, queryExpr, childs[i]);
childs[i].ConOpType = ConnectionOpType.CrossProduct;
childs[i] = new DryadMergeNode(childs[0].OutputPartition.Count, queryExpr, childs[i]);
}
}
else
{
int count = childs[0].OutputPartition.Count;
for (int i = 1; i < childs.Length; ++i)
{
if (childs[i].OutputPartition.Count != count)
{
throw DryadLinqException.Create(HpcLinqErrorCode.HomomorphicApplyNeedsSamePartitionCount,
SR.HomomorphicApplyNeedsSamePartitionCount,
queryExpr);
}
}
}
}
else
{
// Non-homomorphic case.
for (int i = 0; i < childs.Length; ++i)
{
if (childs[i].IsDynamic || childs[i].OutputPartition.Count > 1)
{
childs[i] = new DryadMergeNode(true, false, queryExpr, childs[i]);
}
}
}
DryadQueryNode applyNode = new DryadApplyNode(procLambda, true, queryExpr, childs);
return applyNode;
}
private DryadQueryNode VisitApply(QueryNodeInfo source1,
QueryNodeInfo source2,
LambdaExpression procLambda,
bool perPartition,
bool isFirstOnly,
Expression queryExpr)
{
DryadQueryNode child1 = this.Visit(source1);
DryadQueryNode applyNode;
if (source2 == null)
{
// Unary-apply case:
if (perPartition)
{
//homomorphic
applyNode = this.PromoteConcat(source1, child1, x => new DryadApplyNode(procLambda, queryExpr, x));
}
else
{
//non-homomorphic
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
{
child1 = new DryadMergeNode(true, false, queryExpr, child1);
}
applyNode = new DryadApplyNode(procLambda, queryExpr, child1);
}
}
else
{
// Binary-apply case:
DryadQueryNode child2 = this.Visit(source2);
if (perPartition && isFirstOnly)
{
// The function is left homomorphic:
if (!child2.IsForked && (child1.IsDynamic || child1.OutputPartition.Count > 1))
{
// The normal cases..
if (IsMergeNodeNeeded(child2))
{
if (child1.IsDynamic)
{
child2 = new DryadMergeNode(true, false, queryExpr, child2);
child2.IsForked = true;
}
else
{
// Rather than do full merge and broadcast, which has lots of data-movement
// 1. Tee output2 with output cross-product
// 2. Do a merge-stage which will have input1.nPartition nodes each performing a merge.
// This acheives a distribution of the entire input2 to the Apply nodes with least data-movement.
child2 = new DryadTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
child2.ConOpType = ConnectionOpType.CrossProduct;
child2 = new DryadMergeNode(child1.OutputPartition.Count, queryExpr, child2);
}
}
else
{
// the right-data is alread a single partition, so just tee it.
// this will provide a copy to each of the apply nodes.
child2 = new DryadTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
}
}
else
{
// Less common cases..
// a full merge of the right-data may be necessary.
if (child2.IsDynamic || child2.OutputPartition.Count > 1)
{
child2 = new DryadMergeNode(true, false, queryExpr, child2);
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
{
child2.IsForked = true;
}
}
}
applyNode = new DryadApplyNode(procLambda, queryExpr, child1, child2);
}
else if (perPartition && !isFirstOnly && !child1.IsDynamic && !child2.IsDynamic)
{
// Full homomorphic
// No merging occurs.
// NOTE: We generally expect that both the left and right datasets have matching partitionCount.
// however, we don't test for it yet as users might know what they are doing, and it makes
// LocalDebug inconsistent as LocalDebug doesn't throw in that situation.
applyNode = new DryadApplyNode(procLambda, queryExpr, child1, child2);
}
else
{
// Non-homomorphic
// Full merges of both data sets is necessary.
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
{
child1 = new DryadMergeNode(true, false, queryExpr, child1);
}
if (child2.IsDynamic || child2.OutputPartition.Count > 1)
{
child2 = new DryadMergeNode(true, false, queryExpr, child2);
}
applyNode = new DryadApplyNode(procLambda, queryExpr, child1, child2);
}
}
return applyNode;
}
internal DryadQueryNode InsertTee(bool isForked)
{
if (this.OutputArity != 1)
{
//@@TODO: this should not be reachable. could change to Assert/InvalidOpEx
throw new DryadLinqException(HpcLinqErrorCode.Internal, SR.CannotAddTeeToNode);
}
List<DryadQueryNode> pnodes = new List<DryadQueryNode>(this.Parents);
this.Parents.Clear();
DryadTeeNode teeNode = new DryadTeeNode(this.OutputTypes[0], isForked, this.QueryExpression, this);
teeNode.m_uniqueId = this.m_uniqueId;
teeNode.Parents.AddRange(pnodes);
DryadQueryNode oldNode = this.OutputNode;
foreach (DryadQueryNode pn in pnodes)
{
pn.UpdateChildren(oldNode, teeNode);
}
return teeNode;
}
internal virtual string Visit(DryadTeeNode node,
CodeMemberMethod vertexMethod,
string[] readerNames,
string[] writerNames)
{
return node.AddVertexCode(vertexMethod, readerNames, writerNames);
}
internal DryadForkNode(LambdaExpression fork,
Expression keysExpr,
Expression queryExpr,
DryadQueryNode child)
: base(QueryNodeType.Fork, child.QueryGen, queryExpr, child)
{
this.m_forkLambda = fork;
this.m_keysExpression = keysExpr;
this.m_opName = "Fork";
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
this.m_keys = null;
this.m_keysIdx = -1;
if (keysExpr != null)
{
this.m_keys = evaluator.Eval(keysExpr);
this.m_keysIdx = HpcLinqObjectStore.Put(m_keys);
}
this.m_partitionCount = child.OutputPartition.Count;
PartitionInfo pinfo = new RandomPartition(child.OutputDataSetInfo.partitionInfo.Count);
this.m_outputDataSetInfo = new DataSetInfo(pinfo, DataSetInfo.NoOrderBy, DataSetInfo.NoDistinct);
this.m_dynamicManager = this.InferDynamicManager();
// Finally, create all the children of this:
if (keysExpr == null)
{
Type forkTupleType = fork.Type.GetGenericArguments()[1];
if (forkTupleType.GetGenericTypeDefinition() == typeof(IEnumerable<>))
{
forkTupleType = forkTupleType.GetGenericArguments()[0];
}
Type[] queryTypeArgs = forkTupleType.GetGenericArguments();
this.m_outputTypes = new Type[queryTypeArgs.Length];
for (int i = 0; i < queryTypeArgs.Length; i++)
{
this.m_outputTypes[i] = queryTypeArgs[i];
DryadQueryNode parentNode = new DryadTeeNode(queryTypeArgs[i], true, queryExpr, this);
}
}
else
{
int forkCnt = ((Array)m_keys).Length;
Type forkType = fork.Type.GetGenericArguments()[0];
this.m_outputTypes = new Type[forkCnt];
for (int i = 0; i < forkCnt; i++)
{
this.m_outputTypes[i] = forkType;
DryadQueryNode parentNode = new DryadTeeNode(forkType, true, queryExpr, this);
}
}
}