internal virtual string Visit(DryadDynamicNode node,
CodeMemberMethod vertexMethod,
string[] readerNames,
string[] writerNames)
{
return node.AddVertexCode(vertexMethod, readerNames, writerNames);
}
internal void AddAggregateNode(DryadQueryNode node)
{
switch (this.m_dynamicManager.ManagerType)
{
case DynamicManagerType.None:
{
DryadDynamicNode dnode = new DryadDynamicNode(DynamicManagerType.FullAggregator, this);
this.m_dynamicManager = new DynamicManager(DynamicManagerType.FullAggregator, dnode);
break;
}
case DynamicManagerType.HashDistributor:
{
DryadQueryNode firstVertex = this.m_dynamicManager.GetVertexNode(0);
DryadDynamicNode dnode = firstVertex as DryadDynamicNode;
if (dnode == null || dnode.DynamicType != DynamicManagerType.FullAggregator)
{
dnode = new DryadDynamicNode(DynamicManagerType.FullAggregator, this);
this.m_dynamicManager.InsertVertexNode(0, dnode);
}
break;
}
case DynamicManagerType.FullAggregator:
{
break;
}
default:
{
//@@TODO: this should not be reachable. could change to Assert/InvalidOpEx
throw new DryadLinqException(HpcLinqErrorCode.Internal,
String.Format(SR.DynamicManagerType,
this.m_dynamicManager.ManagerType));
}
}
((DryadDynamicNode)this.m_dynamicManager.GetVertexNode(0)).AddNode(node);
}
private void Initialize(LambdaExpression keySelectExpr,
Expression comparerExpr,
bool isDescending,
bool isTemp,
DryadQueryNode child,
Expression queryExpr)
{
this.m_opName = (keySelectExpr == null) ? "Merge" : "MergeSort";
this.m_keySelectExpression = keySelectExpr;
this.m_comparerExpression = comparerExpr;
this.m_isDescending = isDescending;
this.m_isTemp = isTemp;
if (keySelectExpr != null)
{
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
{
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
queryExpr);
}
}
this.m_comparer = null;
this.m_comparerIdx = -1;
if (comparerExpr != null)
{
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
this.m_comparer = evaluator.Eval(comparerExpr);
this.m_comparerIdx = HpcLinqObjectStore.Put(this.m_comparer);
}
this.m_dynamicManager = DynamicManager.None;
if (this.OpName == "MergeSort" && StaticConfig.UseSMBAggregation)
{
DryadDynamicNode dnode = new DryadDynamicNode(DynamicManagerType.FullAggregator, this);
this.m_dynamicManager = new DynamicManager(DynamicManagerType.FullAggregator, dnode);
}
DryadQueryNode child1 = child;
if ((child is DryadHashPartitionNode) &&
((DryadHashPartitionNode)child).IsDynamicDistributor)
{
child1 = child.Children[0];
this.Children[0] = child1;
bool found = child1.UpdateParent(child, this);
this.m_dynamicManager = this.m_dynamicManager.CreateManager(DynamicManagerType.HashDistributor);
this.m_dynamicManager.InsertVertexNode(-1, child);
}
DataSetInfo childInfo = child1.OutputDataSetInfo;
PartitionInfo pinfo;
if (child1.ConOpType == ConnectionOpType.CrossProduct)
{
this.m_partitionCount = childInfo.partitionInfo.Count;
pinfo = childInfo.partitionInfo;
}
else
{
this.m_partitionCount = 1;
pinfo = DataSetInfo.OnePartition;
}
DistinctInfo dinfo = childInfo.distinctInfo;
OrderByInfo oinfo = DataSetInfo.NoOrderBy;
if (this.OpName == "MergeSort")
{
Type[] typeArgs = this.m_keySelectExpression.Type.GetGenericArguments();
oinfo = OrderByInfo.Create(this.m_keySelectExpression, this.m_comparer, this.m_isDescending, typeArgs[1]);
}
this.m_outputDataSetInfo = new DataSetInfo(pinfo, oinfo, dinfo);
this.m_isSplitting = (((child is DryadHashPartitionNode) &&
((DryadHashPartitionNode)child).IsDynamicDistributor) ||
((child is DryadRangePartitionNode) &&
((DryadRangePartitionNode)child).IsDynamicDistributor));
}
internal DryadAggregateNode(string opName,
Expression seedExpr,
LambdaExpression funcLambda,
LambdaExpression combinerLambda,
LambdaExpression resultLambda,
int stage,
bool isQuery,
Expression queryExpr,
DryadQueryNode child,
bool functionIsExpensive)
: base(QueryNodeType.Aggregate, child.QueryGen, queryExpr, child)
{
this.m_seedExpression = seedExpr;
this.m_funcLambda = funcLambda;
this.m_combinerLambda = combinerLambda;
this.m_resultLambda = resultLambda;
this.m_stage = stage;
this.m_isQuery = isQuery;
this.m_opName = opName;
this.m_seedValue = null;
this.m_seedIdx = -1;
if (seedExpr != null)
{
this.m_seedValue = ExpressionSimplifier.Evaluate(seedExpr);
if (!seedExpr.Type.IsPrimitive)
{
this.m_seedIdx = HpcLinqObjectStore.Put(m_seedValue);
}
}
if (stage != 3)
{
this.m_partitionCount = child.OutputDataSetInfo.partitionInfo.Count;
this.m_outputDataSetInfo = new DataSetInfo();
this.m_outputDataSetInfo.partitionInfo = new RandomPartition(this.m_partitionCount);
this.m_dynamicManager = this.InferDynamicManager();
}
else
{
this.m_partitionCount = 1;
this.m_outputDataSetInfo = new DataSetInfo();
if (functionIsExpensive)
{
DryadDynamicNode dnode = new DryadDynamicNode(DynamicManagerType.FullAggregator, child);
this.m_dynamicManager = new DynamicManager(DynamicManagerType.FullAggregator, dnode);
this.m_dynamicManager.AggregationLevels = 2;
}
else
{
this.m_dynamicManager = DynamicManager.None;
}
}
}
