internal virtual string Visit(DLinqTeeNode node,
CodeMemberMethod vertexMethod,
string[] readerNames,
string[] writerNames)
{
return node.AddVertexCode(vertexMethod, readerNames, writerNames);
}
private DLinqQueryNode VisitMultiApply(QueryNodeInfo source,
LambdaExpression procLambda,
bool perPartition,
bool isFirstOnly,
MethodCallExpression queryExpr)
{
DLinqQueryNode[] childs = new DLinqQueryNode[source.Children.Count];
for (int i = 0; i < source.Children.Count; ++i)
{
childs[i] = this.Visit(source.Children[i].Child);
}
bool isDynamic = childs.Any(x => x.IsDynamic);
if (perPartition && !isDynamic)
{
// Homomorphic case.
if (isFirstOnly)
{
for (int i = 1; i < childs.Length; ++i)
{
childs[i] = new DLinqTeeNode(childs[i].OutputTypes[0], true, queryExpr, childs[i]);
childs[i].ConOpType = ConnectionOpType.CrossProduct;
childs[i] = new DLinqMergeNode(childs[0].OutputPartition.Count, queryExpr, childs[i]);
}
}
else
{
int count = childs[0].OutputPartition.Count;
for (int i = 1; i < childs.Length; ++i)
{
if (childs[i].OutputPartition.Count != count)
{
throw DryadLinqException.Create(DryadLinqErrorCode.HomomorphicApplyNeedsSamePartitionCount,
SR.HomomorphicApplyNeedsSamePartitionCount,
queryExpr);
}
}
}
}
else
{
// Non-homomorphic case.
for (int i = 0; i < childs.Length; ++i)
{
if (childs[i].IsDynamic || childs[i].OutputPartition.Count > 1)
{
childs[i] = new DLinqMergeNode(true, queryExpr, childs[i]);
}
}
}
DLinqQueryNode applyNode = new DLinqApplyNode(procLambda, true, queryExpr, childs);
return applyNode;
}
internal DLinqQueryNode InsertTee(bool isForked)
{
if (this.OutputArity != 1)
{
throw new DryadLinqException(DryadLinqErrorCode.Internal, SR.CannotAddTeeToNode);
}
List<DLinqQueryNode> pnodes = new List<DLinqQueryNode>(this.Parents);
this.Parents.Clear();
DLinqTeeNode teeNode = new DLinqTeeNode(this.OutputTypes[0], isForked, this.QueryExpression, this);
teeNode.m_uniqueId = this.m_uniqueId;
teeNode.Parents.AddRange(pnodes);
DLinqQueryNode oldNode = this.OutputNode;
foreach (DLinqQueryNode pn in pnodes)
{
pn.UpdateChildren(oldNode, teeNode);
}
return teeNode;
}
private DLinqQueryNode VisitApply(QueryNodeInfo source1,
QueryNodeInfo source2,
LambdaExpression procLambda,
bool perPartition,
bool isFirstOnly,
Expression queryExpr)
{
DLinqQueryNode child1 = this.Visit(source1);
DLinqQueryNode applyNode;
if (source2 == null)
{
// Unary-apply case:
if (perPartition)
{
// homomorphic
applyNode = this.PromoteConcat(source1, child1, x => new DLinqApplyNode(procLambda, queryExpr, x));
}
else
{
// non-homomorphic
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
{
child1 = new DLinqMergeNode(true, queryExpr, child1);
}
applyNode = new DLinqApplyNode(procLambda, queryExpr, child1);
}
}
else
{
// Binary-apply case:
DLinqQueryNode child2 = this.Visit(source2);
if (perPartition && isFirstOnly)
{
// The function is left homomorphic:
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
{
// The normal cases..
if (IsMergeNodeNeeded(child2))
{
if (child1.IsDynamic)
{
child2 = new DLinqMergeNode(true, queryExpr, child2);
child2.IsForked = true;
}
else
{
// Rather than do full merge and broadcast, which has lots of data-movement
// 1. Tee output2 with output cross-product
// 2. Do a merge-stage which will have input1.nPartition nodes each performing a merge.
// This acheives a distribution of the entire input2 to the Apply nodes with least data-movement.
child2 = new DLinqTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
child2.ConOpType = ConnectionOpType.CrossProduct;
child2 = new DLinqMergeNode(child1.OutputPartition.Count, queryExpr, child2);
}
}
else
{
// the right-data is alread a single partition, so just tee it.
// this will provide a copy to each of the apply nodes.
child2 = new DLinqTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
}
}
else
{
// a full merge of the right-data may be necessary.
if (child2.IsDynamic || child2.OutputPartition.Count > 1)
{
child2 = new DLinqMergeNode(true, queryExpr, child2);
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
{
child2.IsForked = true;
}
}
}
applyNode = new DLinqApplyNode(procLambda, queryExpr, child1, child2);
}
else if (perPartition && !isFirstOnly && !child1.IsDynamic && !child2.IsDynamic)
{
// Full homomorphic
// No merging occurs.
// NOTE: We generally expect that both the left and right datasets have matching partitionCount.
// however, we don't test for it yet as users might know what they are doing, and it makes
// LocalDebug inconsistent as LocalDebug doesn't throw in that situation.
applyNode = new DLinqApplyNode(procLambda, queryExpr, child1, child2);
}
else
{
// Non-homomorphic
// Full merges of both data sets is necessary.
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
{
child1 = new DLinqMergeNode(true, queryExpr, child1);
}
if (child2.IsDynamic || child2.OutputPartition.Count > 1)
{
child2 = new DLinqMergeNode(true, queryExpr, child2);
}
applyNode = new DLinqApplyNode(procLambda, queryExpr, child1, child2);
}
}
return applyNode;
}
internal DLinqForkNode(LambdaExpression fork,
Expression keysExpr,
Expression queryExpr,
DLinqQueryNode child)
: base(QueryNodeType.Fork, child.QueryGen, queryExpr, child)
{
this.m_forkLambda = fork;
this.m_keysExpression = keysExpr;
this.m_opName = "Fork";
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
this.m_keys = null;
this.m_keysIdx = -1;
if (keysExpr != null)
{
this.m_keys = evaluator.Eval(keysExpr);
this.m_keysIdx = DryadLinqObjectStore.Put(m_keys);
}
this.m_partitionCount = child.OutputPartition.Count;
PartitionInfo pinfo = new RandomPartition(child.OutputDataSetInfo.partitionInfo.Count);
this.m_outputDataSetInfo = new DataSetInfo(pinfo, DataSetInfo.NoOrderBy, DataSetInfo.NoDistinct);
this.m_dynamicManager = this.InferDynamicManager();
// Finally, create all the children of this:
if (keysExpr == null)
{
Type forkTupleType = fork.Type.GetGenericArguments()[1];
if (forkTupleType.GetGenericTypeDefinition() == typeof(IEnumerable<>))
{
forkTupleType = forkTupleType.GetGenericArguments()[0];
}
Type[] queryTypeArgs = forkTupleType.GetGenericArguments();
this.m_outputTypes = new Type[queryTypeArgs.Length];
for (int i = 0; i < queryTypeArgs.Length; i++)
{
this.m_outputTypes[i] = queryTypeArgs[i];
DLinqQueryNode parentNode = new DLinqTeeNode(queryTypeArgs[i], true, queryExpr, this);
}
}
else
{
int forkCnt = ((Array)m_keys).Length;
Type forkType = fork.Type.GetGenericArguments()[0];
this.m_outputTypes = new Type[forkCnt];
for (int i = 0; i < forkCnt; i++)
{
this.m_outputTypes[i] = forkType;
DLinqQueryNode parentNode = new DLinqTeeNode(forkType, true, queryExpr, this);
}
}
}