Beispiel #1
0
        // Evaluate the expression in the current local execution context.
        internal static object Evaluate(Expression expr)
        {
            Type qType = typeof(ExpressionSimplifier <>).MakeGenericType(expr.Type);
            ExpressionSimplifier evaluator = (ExpressionSimplifier)Activator.CreateInstance(qType);

            return(evaluator.EvalBoxed(expr));
        }
Beispiel #2
0
 internal override Expression Visit(Expression expr)
 {
     if (expr == null)
     {
         return(expr);
     }
     if (HpcLinqExpression.IsConstant(expr))
     {
         object val = ExpressionSimplifier.Evaluate(expr);
         if (val is IQueryable)
         {
             QueryNodeInfo nodeInfo;
             if (this.m_referencedQueryMap.TryGetValue(((IQueryable)val).Expression, out nodeInfo))
             {
                 string name = "side__" + this.m_idx;
                 this.m_idx++;
                 this.m_referencedQueries.Add(new Pair <string, DryadQueryNode>(name, nodeInfo.queryNode));
                 return(Expression.Parameter(expr.Type, name));
             }
             throw new DryadLinqException(HpcLinqErrorCode.UnhandledQuery,
                                          String.Format(SR.UnhandledQuery, HpcLinqExpression.Summarize(expr)));
         }
         return(expr);
     }
     return(base.Visit(expr));
 }
Beispiel #3
0
 internal override Expression Visit(Expression expr)
 {
     if (expr == null)
     {
         return(expr);
     }
     if (HpcLinqExpression.IsConstant(expr))
     {
         object val = ExpressionSimplifier.Evaluate(expr);
         if (val is IQueryable)
         {
             this.m_querySet.Add(((IQueryable)val).Expression);
         }
         return(expr);
     }
     return(base.Visit(expr));
 }
Beispiel #4
0
        private DryadQueryNode VisitAssumeRangePartition(QueryNodeInfo source,
                                                         LambdaExpression keySelectExpr,
                                                         Expression keysExpr,
                                                         Expression comparerExpr,
                                                         Expression isDescendingExpr,
                                                         Expression queryExpr)
        {
            DryadQueryNode child = this.Visit(source);

            Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
            if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
            {
                throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
                                              String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
                                              queryExpr);
            }

            object comparer = null;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                comparer = evaluator.Eval(comparerExpr);
            }

            object keys = null;
            if (keysExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                keys = evaluator.Eval(keysExpr);
            }

            //count the number of keys provided.
            if (keys != null)
            {
                int nSeparators = 0;
                var ie = ((IEnumerable)keys).GetEnumerator();
                while (ie.MoveNext())
                {
                    nSeparators++;
                }

                if (!child.IsDynamic && nSeparators != child.PartitionCount - 1)
                {
                    throw DryadLinqException.Create(
                        HpcLinqErrorCode.BadSeparatorCount,
                        String.Format(SR.BadSeparatorCount, nSeparators, child.PartitionCount - 1),
                        queryExpr);
                }
            }

            bool? isDescending = null;
            if (isDescendingExpr != null)
            {
                ExpressionSimplifier<bool> evaluator = new ExpressionSimplifier<bool>();
                isDescending = evaluator.Eval(isDescendingExpr);
            }

            DataSetInfo outputInfo = new DataSetInfo(child.OutputDataSetInfo);
            outputInfo.partitionInfo = PartitionInfo.CreateRange(keySelectExpr,
                                                                 keys,
                                                                 comparer,
                                                                 isDescending,
                                                                 child.OutputPartition.Count,
                                                                 keyType);
            child.OutputDataSetInfo = outputInfo;
            return child;
        }
Beispiel #5
0
        private DryadQueryNode VisitAssumeOrderBy(QueryNodeInfo source,
                                                  LambdaExpression keySelectExpr,
                                                  Expression comparerExpr,
                                                  Expression isDescendingExpr,
                                                  Expression queryExpr)
        {
            DryadQueryNode child = this.Visit(source);

            Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
            if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
            {
                throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
                                              String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
                                              queryExpr);
            }

            object comparer = null;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                comparer = evaluator.Eval(comparerExpr);
            }

            ExpressionSimplifier<bool> bevaluator = new ExpressionSimplifier<bool>();
            bool isDescending = bevaluator.Eval(isDescendingExpr);

            DataSetInfo outputInfo = new DataSetInfo(child.OutputDataSetInfo);
            outputInfo.orderByInfo = OrderByInfo.Create(keySelectExpr, comparer, isDescending, keyType);
            child.OutputDataSetInfo = outputInfo;
            return child;
        }
Beispiel #6
0
        private DryadQueryNode VisitAssumeHashPartition(QueryNodeInfo source,
                                                        LambdaExpression keySelectExpr,
                                                        Expression keysExpr,
                                                        Expression comparerExpr,
                                                        Expression queryExpr)
        {
            DryadQueryNode child = this.Visit(source);

            Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
            if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
            {
                throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
                                              String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType.FullName),
                                              queryExpr);
            }

            object comparer = null;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                comparer = evaluator.Eval(comparerExpr);
            }
            DataSetInfo outputInfo = new DataSetInfo(child.OutputDataSetInfo);
            outputInfo.partitionInfo = PartitionInfo.CreateHash(keySelectExpr,
                                                                child.OutputPartition.Count,
                                                                comparer,
                                                                keyType);
            child.OutputDataSetInfo = outputInfo;
            return child;
        }
Beispiel #7
0
        internal DLinqJoinNode(QueryNodeType nodeType,
                               string opName,
                               LambdaExpression outerKeySelectExpr,
                               LambdaExpression innerKeySelectExpr,
                               LambdaExpression resultSelectExpr,
                               Expression comparerExpr,
                               Expression queryExpr,
                               DLinqQueryNode outerChild,
                               DLinqQueryNode innerChild)
            : base(nodeType, outerChild.QueryGen, queryExpr, outerChild, innerChild)
        {
            Debug.Assert(nodeType == QueryNodeType.Join || nodeType == QueryNodeType.GroupJoin);
            this.m_outerKeySelectExpr = outerKeySelectExpr;
            this.m_innerKeySelectExpr = innerKeySelectExpr;
            this.m_resultSelectExpr = resultSelectExpr;
            this.m_comparerExpr = comparerExpr;
            this.m_opName = opName;
            this.m_attachedPipeline = null;

            this.m_comparer = null;
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                this.m_comparer = evaluator.Eval(comparerExpr);
                this.m_comparerIdx = DryadLinqObjectStore.Put(m_comparer);
            }

            if (StaticConfig.UseMemoryFIFO)
            {
                bool isStateful = this.IsStateful;
                foreach (DLinqQueryNode child in this.Children)
                {
                    if (!(child is DLinqInputNode) &&
                        !child.IsForked &&
                        !(isStateful && child.IsStateful) &&
                        child.PartitionCount > 1)
                    {
                        child.ChannelType = ChannelType.MemoryFIFO;
                    }
                    isStateful = isStateful || child.IsStateful;
                }
            }

            this.m_partitionCount = outerChild.OutputDataSetInfo.partitionInfo.Count;
            this.m_outputDataSetInfo = this.ComputeOutputDataSetInfo();

            this.m_dynamicManager = DynamicManager.None;
        }
Beispiel #8
0
        private DryadQueryNode VisitSetOperation(QueryNodeInfo source1,
                                                 QueryNodeInfo source2,
                                                 QueryNodeType nodeType,
                                                 Expression comparerExpr,
                                                 Expression queryExpr)
        {
            DryadQueryNode child1 = this.Visit(source1);
            DryadQueryNode child2 = this.Visit(source2);
            DryadQueryNode resNode = null;

            Type keyType = child1.OutputTypes[0];
            if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
            {
                throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
                                              string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
                                              queryExpr);
            }

            // The comparer object:
            object comparer = null;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                comparer = evaluator.Eval(comparerExpr);
            }

            LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
            if (child1.IsDynamic || child2.IsDynamic)
            {
                // Well, let us do the simplest thing for now
                child1 = new DryadHashPartitionNode(keySelectExpr,
                                                    null,
                                                    StaticConfig.DefaultPartitionCount,
                                                    queryExpr,
                                                    child1);
                if (IsMergeNodeNeeded(child1))
                {
                    child1 = new DryadMergeNode(false, false, queryExpr, child1);
                }

                child2 = new DryadHashPartitionNode(keySelectExpr,
                                                    null,
                                                    StaticConfig.DefaultPartitionCount,
                                                    queryExpr,
                                                    child2);
                if (IsMergeNodeNeeded(child2))
                {
                    child2 = new DryadMergeNode(false, false, queryExpr, child2);
                }

                resNode = new DryadSetOperationNode(nodeType, nodeType.ToString(), comparerExpr,
                                                    queryExpr, child1, child2);
                return resNode;
            }

            bool isDescending1 = child1.OutputDataSetInfo.orderByInfo.IsDescending;
            bool isDescending2 = child2.OutputDataSetInfo.orderByInfo.IsDescending;

            // Partition child1 and child2 if needed
            if (child1.OutputPartition.ParType == PartitionType.Range &&
                child1.OutputPartition.HasKeys &&
                child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
            {
                if (child2.OutputPartition.ParType != PartitionType.Range ||
                    !child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
                    child1.OutputPartition.IsSamePartition(child2.OutputPartition))
                {
                    // Range distribute child2 using child1's partition
                    child2 = child1.OutputPartition.CreatePartitionNode(keySelectExpr, child2);
                    if (IsMergeNodeNeeded(child2))
                    {
                        child2 = new DryadMergeNode(false, false, queryExpr, child2);
                    }
                }
            }
            else if (child2.OutputPartition.ParType == PartitionType.Range &&
                     child2.OutputPartition.HasKeys &&
                     child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
            {
                // Range distribute child1 using child2's partition
                child1 = child2.OutputPartition.CreatePartitionNode(keySelectExpr, child1);
                if (IsMergeNodeNeeded(child1))
                {
                    child1 = new DryadMergeNode(false, false, queryExpr, child1);
                }
            }
            else if (child1.OutputPartition.ParType == PartitionType.Hash &&
                     child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
            {
                if (child2.OutputPartition.ParType != PartitionType.Hash ||
                    !child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
                    !child1.OutputPartition.IsSamePartition(child2.OutputPartition))
                {
                    // Hash distribute child2:
                    child2 = new DryadHashPartitionNode(keySelectExpr,
                                                        comparerExpr,
                                                        child1.OutputPartition.Count,
                                                        queryExpr,
                                                        child2);
                    if (IsMergeNodeNeeded(child2))
                    {
                        child2 = new DryadMergeNode(false, false, queryExpr, child2);
                    }
                }
            }
            else if (child2.OutputPartition.ParType == PartitionType.Hash &&
                     child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
            {
                child1 = new DryadHashPartitionNode(keySelectExpr,
                                                    comparerExpr,
                                                    child2.OutputPartition.Count,
                                                    queryExpr,
                                                    child1);
                if (IsMergeNodeNeeded(child1))
                {
                    child1 = new DryadMergeNode(false, false, queryExpr, child1);
                }
            }
            else
            {
                // No luck. Hash distribute both child1 and child2, then perform hash operation
                int parCnt = Math.Max(child1.OutputPartition.Count, child2.OutputPartition.Count);
                if (parCnt > 1)
                {
                    child1 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child1);
                    if (IsMergeNodeNeeded(child1))
                    {
                        child1 = new DryadMergeNode(false, false, queryExpr, child1);
                    }

                    child2 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child2);
                    if (IsMergeNodeNeeded(child2))
                    {
                        child2 = new DryadMergeNode(false, false, queryExpr, child2);
                    }
                }
            }

            // Perform either hash or ordered operation
            string opName = "";
            if (child1.IsOrderedBy(keySelectExpr, comparer))
            {
                if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
                    isDescending1 != isDescending2)
                {
                    // Sort inner if unsorted
                    child2 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending1, queryExpr, child2);
                }
                opName = "Ordered";
            }
            else if (child2.IsOrderedBy(keySelectExpr, comparer))
            {
                if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
                    isDescending1 != isDescending2)
                {
                    // Sort outer if unsorted
                    child1 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending2, queryExpr, child1);
                }
                opName = "Ordered";
            }

            resNode = new DryadSetOperationNode(nodeType, opName + nodeType, comparerExpr, queryExpr, child1, child2);
            return resNode;
        }
Beispiel #9
0
        private DryadQueryNode VisitJoin(QueryNodeInfo outerSource,
                                         QueryNodeInfo innerSource,
                                         QueryNodeType nodeType,
                                         LambdaExpression outerKeySelector,
                                         LambdaExpression innerKeySelector,
                                         LambdaExpression resultSelector,
                                         Expression comparerExpr,
                                         Expression queryExpr)
        {
            DryadQueryNode outerChild = this.Visit(outerSource);
            DryadQueryNode innerChild = this.Visit(innerSource);
            DryadQueryNode joinNode = null;

            Type keyType = outerKeySelector.Type.GetGenericArguments()[1];
            if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
            {
                throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
                                              string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
                                              queryExpr);
            }

            // The comparer object:
            object comparer = null;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                comparer = evaluator.Eval(comparerExpr);
            }

            if (outerChild.IsDynamic || innerChild.IsDynamic)
            {
                // Well, let us do the simplest thing for now
                outerChild = new DryadHashPartitionNode(outerKeySelector,
                                                        comparerExpr,
                                                        StaticConfig.DefaultPartitionCount,
                                                        queryExpr,
                                                        outerChild);
                if (IsMergeNodeNeeded(outerChild))
                {
                    outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
                }

                innerChild = new DryadHashPartitionNode(innerKeySelector,
                                                        comparerExpr,
                                                        StaticConfig.DefaultPartitionCount,
                                                        queryExpr,
                                                        innerChild);
                if (IsMergeNodeNeeded(innerChild))
                {
                    innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
                }

                joinNode = new DryadJoinNode(nodeType,
                                             "Hash" + nodeType,
                                             outerKeySelector,
                                             innerKeySelector,
                                             resultSelector,
                                             comparerExpr,
                                             queryExpr,
                                             outerChild,
                                             innerChild);
                return joinNode;
            }

            bool isOuterDescending = outerChild.OutputDataSetInfo.orderByInfo.IsDescending;
            bool isInnerDescending = innerChild.OutputDataSetInfo.orderByInfo.IsDescending;

            // Partition outer and inner if needed
            if (outerChild.OutputPartition.ParType == PartitionType.Range &&
                outerChild.OutputPartition.HasKeys &&
                outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
            {
                if (innerChild.OutputPartition.ParType != PartitionType.Range ||
                    !innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
                    !outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
                {
                    // Range distribute inner using outer's partition.
                    innerChild = outerChild.OutputPartition.CreatePartitionNode(innerKeySelector, innerChild);
                    if (IsMergeNodeNeeded(innerChild))
                    {
                        innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
                    }
                }
            }
            else if (innerChild.OutputPartition.ParType == PartitionType.Range &&
                     innerChild.OutputPartition.HasKeys &&
                     innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
            {
                // Range distribute outer using inner's partition.
                outerChild = innerChild.OutputPartition.CreatePartitionNode(outerKeySelector, outerChild);
                if (IsMergeNodeNeeded(outerChild))
                {
                    outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
                }
            }
            else if (outerChild.OutputPartition.ParType == PartitionType.Hash &&
                     outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
            {
                if (innerChild.OutputPartition.ParType != PartitionType.Hash ||
                    !innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
                    !outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
                {
                    innerChild = new DryadHashPartitionNode(innerKeySelector,
                                                            comparerExpr,
                                                            outerChild.OutputPartition.Count,
                                                            queryExpr,
                                                            innerChild);
                    if (IsMergeNodeNeeded(innerChild))
                    {
                        innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
                    }
                }
            }
            else if (innerChild.OutputPartition.ParType == PartitionType.Hash &&
                     innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
            {
                outerChild = new DryadHashPartitionNode(outerKeySelector,
                                                        comparerExpr,
                                                        innerChild.OutputPartition.Count,
                                                        queryExpr,
                                                        outerChild);
                if (IsMergeNodeNeeded(outerChild))
                {
                    outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
                }
            }
            else
            {
                // No luck. Hash partition both outer and inner
                int parCnt = Math.Max(outerChild.OutputPartition.Count, innerChild.OutputPartition.Count);
                if (parCnt > 1)
                {
                    outerChild = new DryadHashPartitionNode(outerKeySelector,
                                                            comparerExpr,
                                                            parCnt,
                                                            queryExpr,
                                                            outerChild);
                    if (IsMergeNodeNeeded(outerChild))
                    {
                        outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
                    }

                    innerChild = new DryadHashPartitionNode(innerKeySelector,
                                                            comparerExpr,
                                                            parCnt,
                                                            queryExpr,
                                                            innerChild);
                    if (IsMergeNodeNeeded(innerChild))
                    {
                        innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
                    }
                }
            }

            // Perform either merge or hash join
            string opName = "Hash";
            if (outerChild.IsOrderedBy(outerKeySelector, comparer))
            {
                if (!innerChild.IsOrderedBy(innerKeySelector, comparer) ||
                    isOuterDescending != isInnerDescending)
                {
                    // Sort inner if unsorted
                    innerChild = new DryadOrderByNode(innerKeySelector, comparerExpr,
                                                      isOuterDescending, queryExpr, innerChild);
                }
                opName = "Merge";
            }
            else if (innerChild.IsOrderedBy(innerKeySelector, comparer))
            {
                if (!outerChild.IsOrderedBy(outerKeySelector, comparer) ||
                    isOuterDescending != isInnerDescending)
                {
                    // Sort outer if unsorted
                    outerChild = new DryadOrderByNode(outerKeySelector, comparerExpr,
                                                      isInnerDescending, queryExpr, outerChild);
                }
                opName = "Merge";
            }

            joinNode = new DryadJoinNode(nodeType,
                                         opName + nodeType,
                                         outerKeySelector,
                                         innerKeySelector,
                                         resultSelector,
                                         comparerExpr,
                                         queryExpr,
                                         outerChild,
                                         innerChild);
            return joinNode;
        }
Beispiel #10
0
        private DryadQueryNode VisitGroupBy(QueryNodeInfo source,
                                            LambdaExpression keySelectExpr,
                                            LambdaExpression elemSelectExpr,
                                            LambdaExpression resultSelectExpr,
                                            Expression comparerExpr,
                                            Expression queryExpr)
        {
            DryadQueryNode child = this.Visit(source);

            ExpressionInfo einfo = new ExpressionInfo(keySelectExpr);
            if (einfo.IsExpensive)
            {
                // Any method call that is not tagged as "expensive=false" will be deemed expensive.
                // if the keySelector is expensive, we rewrite the query so that the key-function is invoked only once
                // and the record key passed around via a Pair<TKey,TRecord>.
                // keyFunc becomes pair=>pair.Key
                // elementSelector must be rewritten so that references to (record) become (pair.Value)

                Type[] vkTypes = keySelectExpr.Type.GetGenericArguments();
                Type pairType = typeof(Pair<,>).MakeGenericType(vkTypes[1], vkTypes[0]);
                ParameterExpression pairParam = Expression.Parameter(pairType, "e");

                // Add Select(x => new Pair<K,S>(key(x), x))
                ParameterExpression valueParam = keySelectExpr.Parameters[0];
                Expression body = Expression.New(pairType.GetConstructors()[0], keySelectExpr.Body, valueParam);
                Type delegateType = typeof(Func<,>).MakeGenericType(valueParam.Type, body.Type);
                LambdaExpression selectExpr = Expression.Lambda(delegateType, body, valueParam);
                child = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, child);

                // Change keySelector to e => e.Key
                PropertyInfo keyInfo = pairParam.Type.GetProperty("Key");
                body = Expression.Property(pairParam, keyInfo);
                delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
                keySelectExpr = Expression.Lambda(delegateType, body, pairParam);

                // Add or change elementSelector with e.Value
                PropertyInfo valueInfo = pairParam.Type.GetProperty("Value");
                body = Expression.Property(pairParam, valueInfo);
                if (elemSelectExpr != null)
                {
                    ParameterSubst subst = new ParameterSubst(elemSelectExpr.Parameters[0], body);
                    body = subst.Visit(elemSelectExpr.Body);
                }
                delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
                elemSelectExpr = Expression.Lambda(delegateType, body, pairParam);
            }

            Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
            if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
            {
                throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
                                              string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
                                              queryExpr);
            }
            Type elemType;
            if (elemSelectExpr == null)
            {
                elemType = keySelectExpr.Type.GetGenericArguments()[0];
            }
            else
            {
                elemType = elemSelectExpr.Type.GetGenericArguments()[1];
            }

            // The comparer object:
            object comparer = null;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                comparer = evaluator.Eval(comparerExpr);
            }

            LambdaExpression keySelectExpr1 = keySelectExpr;
            LambdaExpression elemSelectExpr1 = elemSelectExpr;
            LambdaExpression resultSelectExpr1 = resultSelectExpr;
            LambdaExpression seedExpr1 = null;
            LambdaExpression accumulateExpr1 = null;

            List<DecompositionInfo> dInfoList = null;
            if (resultSelectExpr != null)
            {
                dInfoList = Decomposition.GetDecompositionInfoList(resultSelectExpr, this.m_codeGen);
            }

            String groupByOpName = "GroupBy";
            DryadQueryNode groupByNode = child;
            bool isPartitioned = child.OutputPartition.IsPartitionedBy(keySelectExpr, comparer);
            if (dInfoList != null)
            {
                // ** Decomposable GroupBy-Reduce
                // This block creates the first GroupByNode and does some preparation for subsequent nodes.
                if (child.IsOrderedBy(keySelectExpr, comparer))
                {
                    groupByOpName = "OrderedGroupBy";
                }

                int dcnt = dInfoList.Count;
                ParameterExpression keyParam;
                if (resultSelectExpr.Parameters.Count == 1)
                {
                    keyParam = Expression.Parameter(keyType, HpcLinqCodeGen.MakeUniqueName("k"));
                }
                else
                {
                    keyParam = resultSelectExpr.Parameters[0];
                }

                // Seed:
                ParameterExpression param2 = Expression.Parameter(
                                                 elemType, HpcLinqCodeGen.MakeUniqueName("e"));
                Expression zeroExpr = Expression.Constant(0, typeof(int));
                Expression seedBody = zeroExpr;
                if (dcnt != 0)
                {
                    LambdaExpression seed = dInfoList[dcnt-1].Seed;
                    ParameterSubst subst = new ParameterSubst(seed.Parameters[0], param2);
                    seedBody = subst.Visit(seed.Body);
                    for (int i = dcnt - 2; i >= 0; i--)
                    {
                        seed = dInfoList[i].Seed;
                        subst = new ParameterSubst(seed.Parameters[0], param2);
                        Expression firstExpr = subst.Visit(seed.Body);
                        Type newPairType = typeof(Pair<,>).MakeGenericType(firstExpr.Type, seedBody.Type);
                        seedBody = Expression.New(newPairType.GetConstructors()[0], firstExpr, seedBody);
                    }
                }
                LambdaExpression seedExpr = Expression.Lambda(seedBody, param2);

                // Accumulate:
                ParameterExpression param1 = Expression.Parameter(
                                                 seedBody.Type, HpcLinqCodeGen.MakeUniqueName("a"));
                Expression accumulateBody = zeroExpr;
                if (dcnt != 0)
                {
                    accumulateBody = Decomposition.AccumulateList(param1, param2, dInfoList, 0);
                }
                LambdaExpression accumulateExpr = Expression.Lambda(accumulateBody, param1, param2);

                // Now prepare for the merge-aggregator and/or in the secondary group-by.
                // keySelectExpr1: e => e.Key
                Type reducerResType = typeof(Pair<,>).MakeGenericType(keyParam.Type, accumulateBody.Type);
                ParameterExpression reducerResParam = Expression.Parameter(reducerResType, "e");
                PropertyInfo keyInfo = reducerResParam.Type.GetProperty("Key");
                Expression body = Expression.Property(reducerResParam, keyInfo);
                Type delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
                keySelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);

                // elemSelectExpr1: e => e.Value
                PropertyInfo valueInfo = reducerResParam.Type.GetProperty("Value");
                body = Expression.Property(reducerResParam, valueInfo);
                delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
                elemSelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);

                // SeedExpr1
                param2 = Expression.Parameter(elemSelectExpr1.Body.Type,
                                              HpcLinqCodeGen.MakeUniqueName("e"));
                seedExpr1 = Expression.Lambda(param2, param2);

                // AccumulateExpr1
                Expression recursiveAccumulateBody = zeroExpr;
                if (dcnt != 0)
                {
                    recursiveAccumulateBody = Decomposition.RecursiveAccumulateList(param1, param2, dInfoList, 0);
                }
                accumulateExpr1 = Expression.Lambda(recursiveAccumulateBody, param1, param2);

                // resultSelectExpr1
                resultSelectExpr1 = null;

                // The first groupByNode.
                // If the input was already correctly partitioned, this will be the only groupByNode.
                bool isPartial = StaticConfig.GroupByLocalAggregationIsPartial && !isPartitioned;
                groupByNode = new DryadGroupByNode(
                                       groupByOpName, keySelectExpr, elemSelectExpr, null,
                                       seedExpr, accumulateExpr, accumulateExpr1, comparerExpr,
                                       isPartial, queryExpr, child);
            }
            else
            {
                // Can't do partial aggregation.
                // Use sort, mergesort, and ordered groupby, if TKey implements IComparable.
                if ((comparer != null && TypeSystem.IsComparer(comparer, keyType)) ||
                    (comparer == null && TypeSystem.HasDefaultComparer(keyType)))
                {
                    if (!child.IsOrderedBy(keySelectExpr, comparer))
                    {
                        groupByNode = new DryadOrderByNode(keySelectExpr, comparerExpr, true, queryExpr, child);
                    }
                    groupByOpName = "OrderedGroupBy";
                }

                // Add a GroupByNode if it is partitioned or has elementSelector.
                // If the input was already correctly partitioned, this will be the only groupByNode.
                if (isPartitioned)
                {
                    groupByNode = new DryadGroupByNode(groupByOpName,
                                                       keySelectExpr,
                                                       elemSelectExpr,
                                                       resultSelectExpr,
                                                       null,  // seed
                                                       null,  // accumulate
                                                       null,  // recursiveAccumulate
                                                       comparerExpr,
                                                       false, // isPartial
                                                       queryExpr,
                                                       groupByNode);
                }
                else if (elemSelectExpr != null)
                {
                    // Local GroupBy without resultSelector:
                    groupByNode = new DryadGroupByNode(groupByOpName,
                                                       keySelectExpr,
                                                       elemSelectExpr,
                                                       null,  // resultSelect
                                                       null,  // seed
                                                       null,  // accumulate
                                                       null,  // recursiveAccumulate
                                                       comparerExpr,
                                                       StaticConfig.GroupByLocalAggregationIsPartial,  // isPartial
                                                       queryExpr,
                                                       groupByNode);

                    // keySelectExpr1: g => g.Key
                    ParameterExpression groupParam = Expression.Parameter(groupByNode.OutputTypes[0], "g");
                    PropertyInfo keyInfo = groupParam.Type.GetProperty("Key");
                    Expression body = Expression.Property(groupParam, keyInfo);
                    Type delegateType = typeof(Func<,>).MakeGenericType(groupParam.Type, body.Type);
                    keySelectExpr1 = Expression.Lambda(delegateType, body, groupParam);

                    // No elementSelector
                    elemSelectExpr1 = null;

                    // resultSelectExpr1
                    ParameterExpression keyParam;
                    Type groupType = typeof(IEnumerable<>).MakeGenericType(groupByNode.OutputTypes[0]);
                    groupParam = Expression.Parameter(groupType, HpcLinqCodeGen.MakeUniqueName("g"));
                    if (resultSelectExpr == null)
                    {
                        // resultSelectExpr1: (k, g) => MakeDryadLinqGroup(k, g)
                        keyParam = Expression.Parameter(keySelectExpr1.Body.Type, HpcLinqCodeGen.MakeUniqueName("k"));
                        MethodInfo groupingInfo = typeof(HpcLinqEnumerable).GetMethod("MakeHpcLinqGroup");
                        groupingInfo = groupingInfo.MakeGenericMethod(keyParam.Type, elemType);
                        body = Expression.Call(groupingInfo, keyParam, groupParam);
                    }
                    else
                    {
                        // resultSelectExpr1: (k, g) => resultSelectExpr(k, FlattenGroups(g))
                        keyParam = resultSelectExpr.Parameters[0];
                        MethodInfo flattenInfo = typeof(HpcLinqEnumerable).GetMethod("FlattenGroups");
                        flattenInfo = flattenInfo.MakeGenericMethod(keyParam.Type, elemType);
                        Expression groupExpr = Expression.Call(flattenInfo, groupParam);
                        ParameterSubst subst = new ParameterSubst(resultSelectExpr.Parameters[1], groupExpr);
                        body = subst.Visit(resultSelectExpr.Body);
                    }
                    delegateType = typeof(Func<,,>).MakeGenericType(keyParam.Type, groupParam.Type, body.Type);
                    resultSelectExpr1 = Expression.Lambda(delegateType, body, keyParam, groupParam);
                }
            }

            // At this point, the first GroupByNode has been created.
            DryadMergeNode mergeNode = null;
            DryadQueryNode groupByNode1 = groupByNode;
            if (!isPartitioned)
            {
                // Create HashPartitionNode, MergeNode, and second GroupByNode

                // Note, if we are doing decomposable-GroupByReduce, there is still some work to go after this
                //   - attach the combiner to the first merge-node
                //   - attach the combiner to the merge-node
                //   - attach finalizer to second GroupBy
                int parCount = (groupByNode.IsDynamic) ? StaticConfig.DefaultPartitionCount : groupByNode.OutputPartition.Count;
                bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
                DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr1, comparerExpr, parCount,
                                                                      isDynamic, queryExpr, groupByNode);

                // Create the Merge Node
                if (groupByOpName == "OrderedGroupBy")
                {
                    // Mergesort with the same keySelector of the hash partition
                    mergeNode = new DryadMergeNode(keySelectExpr1, comparerExpr, true, false, queryExpr, hdistNode);
                }
                else
                {
                    // Random merge
                    mergeNode = new DryadMergeNode(false, false, queryExpr, hdistNode);
                }
                groupByNode1 = new DryadGroupByNode(groupByOpName, keySelectExpr1, elemSelectExpr1,
                                                    resultSelectExpr1, seedExpr1, accumulateExpr1,
                                                    accumulateExpr1, comparerExpr, false, queryExpr,
                                                    mergeNode);
            }

            // Final tidy-up for decomposable GroupBy-Reduce pattern.
            //   - attach combiner to first GroupByNode
            //   - attache combiner to MergeNode as an aggregator
            //   - build a SelectNode to project out results and call finalizer on them.
            if (dInfoList != null)
            {
                // Add dynamic aggregator to the merge-node, if applicable
                if (StaticConfig.GroupByDynamicReduce && !isPartitioned)
                {
                    mergeNode.AddAggregateNode(groupByNode1);
                }

                // Add the final Select node
                Type keyResultPairType = typeof(Pair<,>).MakeGenericType(keyType, seedExpr1.Body.Type);
                ParameterExpression keyResultPairParam = Expression.Parameter(keyResultPairType,
                                                                              HpcLinqCodeGen.MakeUniqueName("e"));
                PropertyInfo valuePropInfo_1 = keyResultPairType.GetProperty("Value");
                Expression combinedValueExpr = Expression.Property(keyResultPairParam, valuePropInfo_1);

                // First, build the combinerList
                int dcnt = dInfoList.Count;
                Expression[] combinerList = new Expression[dcnt];
                for (int i = 0; i < dcnt; i++)
                {
                    if (i + 1 == dcnt)
                    {
                        combinerList[i] = combinedValueExpr;
                    }
                    else
                    {
                        PropertyInfo keyPropInfo = combinedValueExpr.Type.GetProperty("Key");
                        combinerList[i] = Expression.Property(combinedValueExpr, keyPropInfo);
                        PropertyInfo valuePropInfo = combinedValueExpr.Type.GetProperty("Value");
                        combinedValueExpr = Expression.Property(combinedValueExpr, valuePropInfo);
                    }
                    LambdaExpression finalizerExpr = dInfoList[i].FinalReducer;
                    if (finalizerExpr != null)
                    {
                        ParameterSubst subst = new ParameterSubst(finalizerExpr.Parameters[0], combinerList[i]);
                        combinerList[i] = subst.Visit(finalizerExpr.Body);
                    }
                }

                // Build the funcList
                Expression[] funcList = new Expression[dcnt];
                for (int i = 0; i < dcnt; i++)
                {
                    funcList[i] = dInfoList[i].Func;
                }

                // Apply the substitutions
                CombinerSubst combinerSubst = new CombinerSubst(resultSelectExpr, keyResultPairParam, funcList, combinerList);
                Expression finalizerSelectBody = combinerSubst.Visit();

                // Finally, the Select node
                Type delegateType = typeof(Func<,>).MakeGenericType(keyResultPairType, finalizerSelectBody.Type);
                LambdaExpression selectExpr = Expression.Lambda(delegateType, finalizerSelectBody, keyResultPairParam);
                groupByNode1 = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, groupByNode1);
            }
            return groupByNode1;
        }
Beispiel #11
0
     internal DLinqForkNode(LambdaExpression fork,
                            Expression keysExpr,
                            Expression queryExpr,
                            DLinqQueryNode child)
         : base(QueryNodeType.Fork, child.QueryGen, queryExpr, child)
     {
         this.m_forkLambda = fork;
         this.m_keysExpression = keysExpr;
         this.m_opName = "Fork";
 
         ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
         this.m_keys = null;
         this.m_keysIdx = -1;
         if (keysExpr != null)
         {
             this.m_keys = evaluator.Eval(keysExpr);
             this.m_keysIdx = DryadLinqObjectStore.Put(m_keys);
         }
         
         this.m_partitionCount = child.OutputPartition.Count;
         PartitionInfo pinfo = new RandomPartition(child.OutputDataSetInfo.partitionInfo.Count);
         this.m_outputDataSetInfo = new DataSetInfo(pinfo, DataSetInfo.NoOrderBy, DataSetInfo.NoDistinct);
 
         this.m_dynamicManager = this.InferDynamicManager();
 
         // Finally, create all the children of this:
         if (keysExpr == null)
         {
             Type forkTupleType = fork.Type.GetGenericArguments()[1];
             if (forkTupleType.GetGenericTypeDefinition() == typeof(IEnumerable<>))
             {
                 forkTupleType = forkTupleType.GetGenericArguments()[0];
             }
             Type[] queryTypeArgs = forkTupleType.GetGenericArguments();
             this.m_outputTypes = new Type[queryTypeArgs.Length];
             for (int i = 0; i < queryTypeArgs.Length; i++)
             {
                 this.m_outputTypes[i] = queryTypeArgs[i];
                 DLinqQueryNode parentNode = new DLinqTeeNode(queryTypeArgs[i], true, queryExpr, this);
             }
         }
         else
         {
             int forkCnt = ((Array)m_keys).Length;
             Type forkType = fork.Type.GetGenericArguments()[0];
             this.m_outputTypes = new Type[forkCnt];
             for (int i = 0; i < forkCnt; i++)
             {
                 this.m_outputTypes[i] = forkType;
                 DLinqQueryNode parentNode = new DLinqTeeNode(forkType, true, queryExpr, this);
             }
         }
     }
Beispiel #12
0
        //Creates a "Range distribution" Node
        internal DLinqRangePartitionNode(LambdaExpression keySelectExpr,
                                         LambdaExpression resultSelectExpr,
                                         Expression keysExpr,
                                         Expression comparerExpr,
                                         Expression isDescendingExpr,
                                         Expression countExpr,
                                         Expression queryExpr,
                                         params DLinqQueryNode[] children)
            : base(QueryNodeType.RangePartition, children[0].QueryGen, queryExpr, children)
        {
            this.m_keySelectExpression = keySelectExpr;
            this.m_resultSelectExpression = resultSelectExpr;
            this.m_keysExpression = keysExpr;            
            this.m_comparerExpression = comparerExpr;
            this.m_isDescendingExpression = isDescendingExpr;
            this.m_countExpression = countExpr;            
            this.m_opName = "RangePartition";
            this.m_conOpType = ConnectionOpType.CrossProduct;

            this.m_isDescending = null;
            if (this.m_isDescendingExpression != null)
            {
                ExpressionSimplifier<bool> bevaluator = new ExpressionSimplifier<bool>();
                this.m_isDescending = bevaluator.Eval(isDescendingExpr);
            }

            ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
            this.m_keys = null;
            this.m_keysIdx = -1;
            if (keysExpr != null)
            {
                this.m_keys = evaluator.Eval(keysExpr);
                this.m_keysIdx = DryadLinqObjectStore.Put(m_keys);
            }

            this.m_comparer = null;
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                this.m_comparer = evaluator.Eval(comparerExpr);
                this.m_comparerIdx = DryadLinqObjectStore.Put(m_comparer);
            }

            this.m_count = 1;
            if (countExpr != null)
            {
                ExpressionSimplifier<int> ievaluator = new ExpressionSimplifier<int>();
                this.m_count = ievaluator.Eval(countExpr);
            }
            
            this.m_partitionCount = this.Children[0].OutputDataSetInfo.partitionInfo.Count;

            DataSetInfo childInfo = this.Children[0].OutputDataSetInfo;
            Type keyType = this.m_keySelectExpression.Type.GetGenericArguments()[1];
            PartitionInfo pInfo = PartitionInfo.CreateRange(this.m_keySelectExpression,
                                                            this.m_keys,
                                                            this.m_comparer,
                                                            this.m_isDescending,
                                                            this.m_count,
                                                            keyType);
            OrderByInfo oinfo = childInfo.orderByInfo;
            DistinctInfo dinfo = childInfo.distinctInfo;
            this.m_outputDataSetInfo = new DataSetInfo(pInfo, oinfo, dinfo);
            this.m_dynamicManager = this.InferDynamicManager();
        }
Beispiel #13
0
        internal DLinqHashPartitionNode(LambdaExpression keySelectExpr,
                                        LambdaExpression resultSelectExpr,
                                        Expression comparerExpr,
                                        int count,
                                        bool isDynamic,
                                        Expression queryExpr,
                                        DLinqQueryNode child)
            : base(QueryNodeType.HashPartition, child.QueryGen, queryExpr, child)
        {
            this.m_keySelectExpr = keySelectExpr;
            this.m_resultSelectExpr = resultSelectExpr;
            this.m_parCount = count;
            this.m_isDynamic = isDynamic;
            this.m_comparerExpression = comparerExpr;
            this.m_opName = "HashPartition";
            this.m_conOpType = ConnectionOpType.CrossProduct;

            this.m_comparer = null;
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                this.m_comparer = evaluator.Eval(comparerExpr);
                this.m_comparerIdx = DryadLinqObjectStore.Put(this.m_comparer);
            }

            this.m_partitionCount = child.OutputDataSetInfo.partitionInfo.Count;

            Type keyType = child.OutputTypes[0];
            if (this.m_keySelectExpr != null)
            {
                keyType = this.m_keySelectExpr.Type.GetGenericArguments()[1];
            }
            DataSetInfo childInfo = child.OutputDataSetInfo;
            PartitionInfo pInfo = PartitionInfo.CreateHash(this.m_keySelectExpr,
                                                           this.m_parCount,
                                                           this.m_comparer,
                                                           keyType);
            OrderByInfo oinfo = childInfo.orderByInfo;
            DistinctInfo dinfo = childInfo.distinctInfo;
            this.m_outputDataSetInfo = new DataSetInfo(pInfo, oinfo, dinfo);

            this.m_dynamicManager = this.InferDynamicManager();
        }
Beispiel #14
0
        private void Initialize(LambdaExpression keySelectExpr,
                                Expression comparerExpr,
                                bool isDescending,
                                DLinqQueryNode child,
                                Expression queryExpr)
        {
            this.m_opName = (keySelectExpr == null) ? "Merge" : "MergeSort";
            this.m_keySelectExpr = keySelectExpr;
            this.m_comparerExpr = comparerExpr;
            this.m_isDescending = isDescending;

            if (keySelectExpr != null)
            {
                Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
                if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
                {
                    throw DryadLinqException.Create(DryadLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
                                                    String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
                                                    queryExpr);
                }
            }

            this.m_comparer = null;
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                this.m_comparer = evaluator.Eval(comparerExpr);
                this.m_comparerIdx = DryadLinqObjectStore.Put(this.m_comparer);
            }

            this.m_dynamicManager = DynamicManager.None;
            if (this.OpName == "MergeSort" && StaticConfig.UseSMBAggregation)
            {
                DLinqDynamicNode dnode = new DLinqDynamicNode(DynamicManagerType.FullAggregator, this);
                this.m_dynamicManager = new DynamicManager(DynamicManagerType.FullAggregator, dnode);
            }

            DLinqQueryNode child1 = child;
            if ((child is DLinqHashPartitionNode) &&
                ((DLinqHashPartitionNode)child).IsDynamicDistributor)
            {
                child1 = child.Children[0];
                this.Children[0] = child1;
                bool found = child1.UpdateParent(child, this);
                this.m_dynamicManager = this.m_dynamicManager.CreateManager(DynamicManagerType.HashDistributor);
                this.m_dynamicManager.InsertVertexNode(-1, child);
            }

            DataSetInfo childInfo = child1.OutputDataSetInfo;
            PartitionInfo pinfo;
            if (child1.ConOpType == ConnectionOpType.CrossProduct)
            {
                this.m_partitionCount = childInfo.partitionInfo.Count;
                pinfo = childInfo.partitionInfo;
            }
            else
            {
                this.m_partitionCount = 1;
                pinfo = DataSetInfo.OnePartition;
            }

            DistinctInfo dinfo = childInfo.distinctInfo;
            OrderByInfo oinfo = DataSetInfo.NoOrderBy;
            if (this.OpName == "MergeSort")
            {
                Type[] typeArgs = this.m_keySelectExpr.Type.GetGenericArguments();
                oinfo = OrderByInfo.Create(this.m_keySelectExpr, this.m_comparer, this.m_isDescending, typeArgs[1]);
            }
            this.m_outputDataSetInfo = new DataSetInfo(pinfo, oinfo, dinfo);

            this.m_isSplitting = (((child is DLinqHashPartitionNode) &&
                                   ((DLinqHashPartitionNode)child).IsDynamicDistributor) ||
                                  ((child is DLinqRangePartitionNode) &&
                                   ((DLinqRangePartitionNode)child).IsDynamicDistributor));
        }
Beispiel #15
0
        internal DLinqSetOperationNode(QueryNodeType nodeType,
                                       string opName,
                                       Expression comparerExpr,
                                       Expression queryExpr,
                                       DLinqQueryNode child1,
                                       DLinqQueryNode child2)
            : base(nodeType, child1.QueryGen, queryExpr, child1, child2)
        {
            this.m_isOrdered = opName.StartsWith("Ordered", StringComparison.Ordinal);
            this.m_opName = opName;
            this.m_comparerExpr = comparerExpr;

            this.m_comparer = null;
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                this.m_comparer = evaluator.Eval(comparerExpr);
                this.m_comparerIdx = DryadLinqObjectStore.Put(m_comparer);
            }

            if (StaticConfig.UseMemoryFIFO)
            {
                bool isStateful = this.IsStateful;
                foreach (DLinqQueryNode child in this.Children)
                {
                    if (!(child is DLinqInputNode) &&
                        !child.IsForked &&
                        !(isStateful && child.IsStateful) &&
                        child.PartitionCount > 1)
                    {
                        child.ChannelType = ChannelType.MemoryFIFO;
                    }
                    isStateful = isStateful || child.IsStateful;
                }
            }

            this.m_partitionCount = child1.OutputDataSetInfo.partitionInfo.Count;
            this.m_outputDataSetInfo = new DataSetInfo(child1.OutputDataSetInfo);

            this.m_dynamicManager = DynamicManager.None;
        }
Beispiel #16
0
        internal DLinqContainsNode(Expression valueExpr,
                                   Expression comparerExpr,
                                   Expression queryExpr,
                                   DLinqQueryNode child)
            : base(QueryNodeType.Contains, child.QueryGen, queryExpr, child)
        {
            this.m_valueExpr = valueExpr;
            this.m_comparerExpr = comparerExpr;
            this.m_opName = "Contains";

            this.m_valueIdx = DryadLinqObjectStore.Put(ExpressionSimplifier.Evaluate(valueExpr));
            ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                this.m_comparerIdx = DryadLinqObjectStore.Put(evaluator.Eval(comparerExpr));
            }

            this.m_partitionCount = child.OutputDataSetInfo.partitionInfo.Count;
            this.m_outputDataSetInfo = new DataSetInfo();
            this.m_outputDataSetInfo.partitionInfo = new RandomPartition(this.m_partitionCount);

            this.m_dynamicManager = this.InferDynamicManager();
        }
Beispiel #17
0
        internal DLinqDistinctNode(bool isPartial,
                                   Expression comparerExpr, 
                                   Expression queryExpr, 
                                   DLinqQueryNode child)
            : base(QueryNodeType.Distinct, child.QueryGen, queryExpr, child)
        {
            this.m_isPartial = isPartial;
            this.m_comparerExpr = comparerExpr;
            this.m_opName = "Distinct";
            this.m_attachedPipeline = null;

            this.m_comparer = null;
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                this.m_comparer = evaluator.Eval(comparerExpr);
                this.m_comparerIdx = DryadLinqObjectStore.Put(this.m_comparer);
            }

            this.m_partitionCount = child.OutputDataSetInfo.partitionInfo.Count;

            this.m_outputDataSetInfo = new DataSetInfo(child.OutputDataSetInfo);
            this.m_outputDataSetInfo.distinctInfo = DistinctInfo.Create(this.m_comparer, this.OutputTypes[0]);

            this.m_dynamicManager = this.InferDynamicManager();
        }
Beispiel #18
0
        private DryadQueryNode VisitDistinct(QueryNodeInfo source,
                                             Expression comparerExpr,
                                             Expression queryExpr)
        {
            DryadQueryNode child = this.Visit(source);

            Type keyType = child.OutputTypes[0];
            if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
            {
                throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
                                              string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
                                              queryExpr);
            }

            object comparer = null;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                comparer = evaluator.Eval(comparerExpr);
            }

            LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
            if (!child.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
            {
                if (child.IsDynamic || child.OutputPartition.Count > 1)
                {
                    child = new DryadDistinctNode(true, comparerExpr, queryExpr, child);
                    bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
                    child = new DryadHashPartitionNode(keySelectExpr,
                                                       comparerExpr,
                                                       child.OutputPartition.Count,
                                                       isDynamic,
                                                       queryExpr,
                                                       child);
                    child = new DryadMergeNode(false, false, queryExpr, child);
                }
            }
            DryadQueryNode resNode = new DryadDistinctNode(false, comparerExpr, queryExpr, child);
            return resNode;
        }
Beispiel #19
0
 private DryadQueryNode VisitForkChoose(QueryNodeInfo source,
                                        Expression indexExpr,
                                        Expression queryExpr)
 {
     DryadQueryNode child = this.Visit(source);
     ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
     int index = evaluator.Eval(indexExpr);
     return ((DryadForkNode)child).Parents[index];
 }
 // This constructor is specifically to support SubmitAndWait() calls.
 // It assumes that the Expressions all terminate with a ToStore node.
 internal DryadLinqQueryGen(DryadLinqContext context,
                            VertexCodeGen vertexCodeGen,
                            Expression[] qlist)
 {
     this.m_queryExprs = new Expression[qlist.Length];
     this.m_outputTableUris = new Uri[qlist.Length];
     this.m_serializers = new Expression[qlist.Length];
     this.m_deserializers = new Expression[qlist.Length];
     this.m_isTempOutput = new bool[qlist.Length];
     this.m_context = context;
     for (int i = 0; i < this.m_queryExprs.Length; i++)
     {
         MethodCallExpression mcExpr = qlist[i] as MethodCallExpression;
         
         if (mcExpr != null && mcExpr.Method.Name == ReflectedNames.DLQ_ToStoreInternal)
         {
             this.m_queryExprs[i] = mcExpr.Arguments[0];
             ExpressionSimplifier<Uri> e2 = new ExpressionSimplifier<Uri>();
             this.m_outputTableUris[i] = e2.Eval(mcExpr.Arguments[1]);
             ExpressionSimplifier<bool> e3 = new ExpressionSimplifier<bool>();
             this.m_isTempOutput[i] = e3.Eval(mcExpr.Arguments[2]);
             this.m_serializers[i] = mcExpr.Arguments[3];
             this.m_deserializers[i] = mcExpr.Arguments[4];
         }
         else if (mcExpr != null && mcExpr.Method.Name == ReflectedNames.DLQ_ToStoreInternalAux)
         {
             this.m_queryExprs[i] = mcExpr.Arguments[0];
             ExpressionSimplifier<Uri> e2 = new ExpressionSimplifier<Uri>();
             this.m_outputTableUris[i] = e2.Eval(mcExpr.Arguments[1]);
             ExpressionSimplifier<bool> e3 = new ExpressionSimplifier<bool>();
             this.m_isTempOutput[i] = e3.Eval(mcExpr.Arguments[2]);
         }
         else
         {
             throw new DryadLinqException("Internal error: The method must be " + ReflectedNames.DLQ_ToStoreInternal);
         }
     }
     this.Initialize(vertexCodeGen);
 }
Beispiel #21
0
        private DryadQueryNode VisitHashPartition(QueryNodeInfo source,
                                                  LambdaExpression keySelectExpr,
                                                  Expression comparerExpr,
                                                  Expression countExpr,
                                                  Expression queryExpr)
        {
            DryadQueryNode child = this.Visit(source);
            Type keyType = keySelectExpr.Type.GetGenericArguments()[1];

            if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
            {
                throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
                                              string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
                                              queryExpr);
            }

            bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;

            int nOutputPartitions;
            if (countExpr != null)
            {
                ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
                nOutputPartitions = evaluator.Eval(countExpr);
                isDynamic = false;
            }
            else
            {
                // Note: For MayRTM, isDynamic will never be true.
                nOutputPartitions = (isDynamic) ? 1 : child.OutputPartition.Count;
            }

            DryadQueryNode resNode = new DryadHashPartitionNode(
                                              keySelectExpr, null, comparerExpr, nOutputPartitions,
                                              isDynamic, queryExpr, child);
            resNode = new DryadMergeNode(false, true, queryExpr, resNode);
            return resNode;
        }
        private static DecompositionInfo GetDecompositionInfo(ParameterExpression groupParam,
                                                              MethodCallExpression mcExpr,
                                                              HpcLinqCodeGen codeGen)
        {
            if (mcExpr.Arguments.Count == 0 || mcExpr.Arguments[0] != groupParam)
            {
                return(null);
            }
            for (int i = 1; i < mcExpr.Arguments.Count; i++)
            {
                if (HpcLinqExpression.Contains(groupParam, mcExpr.Arguments[i]))
                {
                    return(null);
                }
            }

            ExpressionSimplifier <object> evaluator = new ExpressionSimplifier <object>();

            Type[] paramTypeArgs  = groupParam.Type.GetGenericArguments();
            Type   sourceElemType = paramTypeArgs[paramTypeArgs.Length - 1];
            Type   resultType     = mcExpr.Type;
            Type   decomposerType = null;

            DecomposableAttribute decomposableAttrib = AttributeSystem.GetDecomposableAttrib(mcExpr);

            if (decomposableAttrib != null)
            {
                decomposerType = decomposableAttrib.DecompositionType;
            }
            else
            {
                MethodInfo mInfo = mcExpr.Method;
                if (mInfo.DeclaringType == typeof(System.Linq.Enumerable) ||
                    mInfo.DeclaringType == typeof(System.Linq.Queryable))
                {
                    // For built-in decomposable operators.
                    switch (mInfo.Name)
                    {
                    case "Count":
                    case "LongCount":
                    {
                        Type       outputType;
                        Expression body;
                        if (mInfo.Name == "Count")
                        {
                            outputType = typeof(Int32);
                            body       = Expression.Constant(1, outputType);
                        }
                        else
                        {
                            outputType = typeof(Int64);
                            body       = Expression.Constant((long)1, outputType);
                        }
                        ParameterExpression param1   = Expression.Parameter(outputType, "a");
                        ParameterExpression param2   = Expression.Parameter(sourceElemType, "e");
                        LambdaExpression    seedExpr = Expression.Lambda(body, param2);
                        body = Expression.AddChecked(param1, body);
                        LambdaExpression accumulateExpr = Expression.Lambda(body, param1, param2);
                        param2 = Expression.Parameter(outputType, "b");
                        body   = Expression.AddChecked(param1, param2);
                        LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2);

                        return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
                    }

                    case "Any":
                    {
                        ParameterExpression param1 = Expression.Parameter(typeof(bool), "a");
                        ParameterExpression param2;
                        Expression          body;
                        if (mcExpr.Arguments.Count == 1)
                        {
                            param2 = Expression.Parameter(sourceElemType, "e");
                            body   = Expression.Constant(true, typeof(bool));
                        }
                        else
                        {
                            LambdaExpression predExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
                            param2 = predExpr.Parameters[0];
                            body   = predExpr.Body;
                        }

                        LambdaExpression seedExpr       = Expression.Lambda(body, param2);
                        LambdaExpression accumulateExpr = Expression.Lambda(Expression.Or(param1, body), param1, param2);
                        param2 = Expression.Parameter(typeof(bool), "b");
                        body   = Expression.Or(param1, param2);
                        LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2);

                        return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
                    }

                    case "All":
                    {
                        ParameterExpression param1   = Expression.Parameter(typeof(bool), "a");
                        LambdaExpression    predExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
                        ParameterExpression param2   = predExpr.Parameters[0];

                        Expression       body           = predExpr.Body;
                        LambdaExpression seedExpr       = Expression.Lambda(body, param2);
                        LambdaExpression accumulateExpr = Expression.Lambda(Expression.And(param1, body), param1, param2);
                        param2 = Expression.Parameter(typeof(bool), "b");
                        body   = Expression.And(param1, param2);
                        LambdaExpression recursiveAccumulateExpr = Expression.Lambda(body, param1, param2);

                        return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
                    }

                    case "First":
                    {
                        ParameterExpression param1 = Expression.Parameter(sourceElemType, "a");
                        ParameterExpression param2 = Expression.Parameter(sourceElemType, "e");

                        LambdaExpression seedExpr                = Expression.Lambda(param2, param2);
                        LambdaExpression accumulateExpr          = Expression.Lambda(param1, param1, param2);
                        LambdaExpression recursiveAccumulateExpr = accumulateExpr;

                        return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
                    }

                    case "Last":
                    {
                        ParameterExpression param1 = Expression.Parameter(sourceElemType, "a");
                        ParameterExpression param2 = Expression.Parameter(sourceElemType, "e");

                        LambdaExpression seedExpr                = Expression.Lambda(param2, param2);
                        LambdaExpression accumulateExpr          = Expression.Lambda(param2, param1, param2);
                        LambdaExpression recursiveAccumulateExpr = accumulateExpr;

                        return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
                    }

                    case "Sum":
                    {
                        ParameterExpression param1;
                        ParameterExpression param2;
                        Expression          arg2;
                        if (mInfo.GetParameters().Length == 1)
                        {
                            param2 = Expression.Parameter(sourceElemType, "e");
                            arg2   = param2;
                        }
                        else
                        {
                            LambdaExpression selectExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
                            param2 = selectExpr.Parameters[0];
                            arg2   = selectExpr.Body;
                        }

                        Expression abody, sbody;
                        if (arg2.Type.IsGenericType)
                        {
                            param1 = Expression.Parameter(arg2.Type.GetGenericArguments()[0], "a");
                            MethodInfo accumulateInfo = typeof(HpcLinqVertex).GetMethod(
                                "SumAccumulate",
                                new Type[] { param1.Type, arg2.Type });
                            sbody = Expression.Constant(0, param1.Type);
                            sbody = Expression.Call(accumulateInfo, sbody, arg2);
                            abody = Expression.Call(accumulateInfo, param1, arg2);
                        }
                        else
                        {
                            param1 = Expression.Parameter(arg2.Type, "a");
                            sbody  = arg2;
                            abody  = Expression.AddChecked(param1, arg2);
                        }

                        LambdaExpression seedExpr       = Expression.Lambda(sbody, param2);
                        LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2);
                        param2 = Expression.Parameter(param1.Type, "b");
                        Expression       rbody = Expression.AddChecked(param1, param2);
                        LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2);
                        Expression       fbody           = Expression.Convert(param1, arg2.Type);
                        LambdaExpression finalReduceExpr = Expression.Lambda(fbody, param1);
                        return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr,
                                                     recursiveAccumulateExpr, finalReduceExpr));
                    }

                    case "Max":
                    case "Min":
                    {
                        ParameterExpression param2;
                        Expression          abody;
                        if (mInfo.GetParameters().Length == 1)
                        {
                            param2 = Expression.Parameter(sourceElemType, "e");
                            abody  = param2;
                        }
                        else
                        {
                            LambdaExpression selectExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
                            param2 = selectExpr.Parameters[0];
                            abody  = selectExpr.Body;
                        }

                        ParameterExpression param1 = Expression.Parameter(abody.Type, "a");
                        Expression          sbody  = abody;
                        MethodInfo          accumulateInfo;
                        string methodName = (mInfo.Name == "Max") ? "MaxAccumulate" : "MinAccumulate";
                        if (mInfo.IsGenericMethod && (mInfo.GetParameters().Length == 1))
                        {
                            accumulateInfo = typeof(HpcLinqVertex).GetMethod(methodName + "Generic");
                            accumulateInfo = accumulateInfo.MakeGenericMethod(sourceElemType);
                        }
                        else
                        {
                            accumulateInfo = typeof(HpcLinqVertex).GetMethod(
                                methodName,
                                new Type[] { param1.Type, abody.Type });
                        }
                        abody = Expression.Call(accumulateInfo, param1, abody);

                        LambdaExpression seedExpr       = Expression.Lambda(sbody, param2);
                        LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2);
                        param2 = Expression.Parameter(param1.Type, "b");
                        Expression       rbody = Expression.Call(accumulateInfo, param1, param2);
                        LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2);
                        return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
                    }

                    case "Aggregate":
                    {
                        ParameterExpression elemParam = Expression.Parameter(sourceElemType, "e");
                        LambdaExpression    accumulateExpr;
                        LambdaExpression    seedExpr;
                        if (mcExpr.Arguments.Count == 2)
                        {
                            accumulateExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
                            seedExpr       = Expression.Lambda(elemParam, elemParam);
                        }
                        else
                        {
                            accumulateExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[2]);
                            object         seedVal = evaluator.Eval(mcExpr.Arguments[1]);
                            Expression     body    = Expression.Constant(seedVal, seedVal.GetType());
                            ParameterSubst subst   = new ParameterSubst(accumulateExpr.Parameters[0], body);
                            body     = subst.Visit(accumulateExpr.Body);
                            seedExpr = Expression.Lambda(body, accumulateExpr.Parameters[1]);
                        }
                        if (!HpcLinqExpression.IsAssociative(accumulateExpr))
                        {
                            return(null);
                        }
                        LambdaExpression recursiveAccumulateExpr = HpcLinqExpression.GetAssociativeCombiner(accumulateExpr);
                        return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, null));
                    }

                    case "Average":
                    {
                        ParameterExpression param2;
                        Expression          abody;
                        if (mInfo.GetParameters().Length == 1)
                        {
                            param2 = Expression.Parameter(sourceElemType, "e");
                            abody  = param2;
                        }
                        else
                        {
                            LambdaExpression selectExpr = HpcLinqExpression.GetLambda(mcExpr.Arguments[1]);
                            param2 = selectExpr.Parameters[0];
                            abody  = selectExpr.Body;
                        }
                        Type aggValueType = abody.Type;
                        if (aggValueType == typeof(int) ||
                            aggValueType == typeof(int?))
                        {
                            aggValueType = typeof(long);
                        }
                        else if (aggValueType == typeof(long?))
                        {
                            aggValueType = typeof(long);
                        }
                        else if (aggValueType == typeof(float) ||
                                 aggValueType == typeof(float?))
                        {
                            aggValueType = typeof(double);
                        }
                        else if (aggValueType == typeof(double?))
                        {
                            aggValueType = typeof(double);
                        }
                        else if (aggValueType == typeof(decimal?))
                        {
                            aggValueType = typeof(decimal);
                        }

                        Type sumAndCountType               = typeof(AggregateValue <>).MakeGenericType(aggValueType);
                        ParameterExpression param1         = Expression.Parameter(sumAndCountType, "a");
                        MethodInfo          accumulateInfo = typeof(HpcLinqVertex).GetMethod(
                            "AverageAccumulate",
                            new Type[] { sumAndCountType, abody.Type });

                        // Seed:
                        Expression sbody = Expression.New(sumAndCountType);
                        sbody = Expression.Call(accumulateInfo, sbody, abody);
                        LambdaExpression seedExpr = Expression.Lambda(sbody, param2);

                        // Accumulate:
                        abody = Expression.Call(accumulateInfo, param1, abody);
                        LambdaExpression accumulateExpr = Expression.Lambda(abody, param1, param2);

                        // RecursiveAccumulate:
                        param2 = Expression.Parameter(param1.Type, "b");
                        PropertyInfo     valueInfo  = sumAndCountType.GetProperty("Value");
                        PropertyInfo     countInfo  = sumAndCountType.GetProperty("Count");
                        Expression       sumExpr1   = Expression.Property(param1, valueInfo);
                        Expression       countExpr1 = Expression.Property(param1, countInfo);
                        Expression       sumExpr2   = Expression.Property(param2, valueInfo);
                        Expression       countExpr2 = Expression.Property(param2, countInfo);
                        Expression       sumExpr    = Expression.AddChecked(sumExpr1, sumExpr2);
                        Expression       countExpr  = Expression.AddChecked(countExpr1, countExpr2);
                        ConstructorInfo  cinfo      = sumAndCountType.GetConstructor(new Type[] { sumExpr.Type, countExpr.Type });
                        Expression       rbody      = Expression.New(cinfo, sumExpr, countExpr);
                        LambdaExpression recursiveAccumulateExpr = Expression.Lambda(rbody, param1, param2);

                        // FinalReduce:
                        if (sumExpr1.Type == typeof(long))
                        {
                            sumExpr1 = Expression.Convert(sumExpr1, typeof(double));
                        }
                        Expression fbody = Expression.Divide(sumExpr1, countExpr1);
                        fbody = Expression.Convert(fbody, resultType);
                        if (resultType.IsGenericType)
                        {
                            Expression zeroExpr = Expression.Constant(0, typeof(long));
                            Expression condExpr = Expression.GreaterThan(countExpr1, zeroExpr);
                            Expression nullExpr = Expression.Constant(null, resultType);
                            fbody = Expression.Condition(condExpr, fbody, nullExpr);
                        }
                        LambdaExpression finalReduceExpr = Expression.Lambda(fbody, param1);
                        return(new DecompositionInfo(mcExpr, seedExpr, accumulateExpr, recursiveAccumulateExpr, finalReduceExpr));
                    }

                    case "Contains":
                    {
                        decomposerType = typeof(ContainsDecomposition <>).MakeGenericType(sourceElemType);
                        break;
                    }

                    case "Distinct":
                    {
                        decomposerType = typeof(DistinctDecomposition <>).MakeGenericType(sourceElemType);
                        break;
                    }

                    default:
                    {
                        return(null);
                    }
                    }
                }
            }

            if (decomposerType == null)
            {
                return(null);
            }

            Type implementedInterface = null;

            Type[] interfaces = decomposerType.GetInterfaces();
            foreach (Type intf in interfaces)
            {
                if (intf.GetGenericTypeDefinition() == typeof(IDecomposable <, ,>))
                {
                    if (implementedInterface != null)
                    {
                        throw new DryadLinqException("Decomposition class can implement only one decomposable interface.");
                    }
                    implementedInterface = intf;
                }
            }

            if (implementedInterface == null ||
                implementedInterface.GetGenericArguments().Length != 3)
            {
                throw new DryadLinqException("Decomposition class " + decomposerType.FullName +
                                             "must implement IDecomposable<,,>");
            }

            // The second type of the implemented interface definition is the accumulatorType.
            Type accumulatorType = implementedInterface.GetGenericArguments()[1];

            // Now check that all the types match up.
            Type decomposerInterface = typeof(IDecomposable <, ,>).MakeGenericType(
                sourceElemType, accumulatorType, resultType);

            if (!decomposerInterface.IsAssignableFrom(decomposerType))
            {
                throw new DryadLinqException("Decomposition class must match the function that it decorates.");
            }
            if (decomposerType.ContainsGenericParameters)
            {
                if (decomposerType.GetGenericArguments().Length != 1 ||
                    !decomposerType.GetGenericArguments()[0].IsGenericParameter)
                {
                    throw new DryadLinqException(decomposerType.Name + " must match the function it annotates.");
                }
                decomposerType = decomposerType.MakeGenericType(sourceElemType);
            }
            if (decomposerType.GetConstructor(Type.EmptyTypes) == null)
            {
                throw new DryadLinqException("Decomposition class must have a default constructor.");
            }

            // Add to the codegen a call of the static Initializer of decomposerType
            Expression[] args = new Expression[mcExpr.Arguments.Count - 1];
            for (int i = 0; i < args.Length; i++)
            {
                args[i] = Expression.Convert(mcExpr.Arguments[i + 1], typeof(object));
            }
            Expression          stateExpr      = Expression.NewArrayInit(typeof(object), args);
            string              decomposerName = codeGen.AddDecompositionInitializer(decomposerType, stateExpr);
            ParameterExpression decomposer     = Expression.Parameter(decomposerType, decomposerName);

            // Seed: TSource => TAccumulate
            MethodInfo          seedInfo1 = decomposerType.GetMethod("Seed");
            ParameterExpression p2        = Expression.Parameter(sourceElemType, "e");
            Expression          sbody1    = Expression.Call(decomposer, seedInfo1, p2);
            LambdaExpression    seedExpr1 = Expression.Lambda(sbody1, p2);

            // Accumulate: (TAccumulate, TSource) => TAccumulate
            MethodInfo          accumulateInfo1 = decomposerType.GetMethod("Accumulate");
            ParameterExpression p1              = Expression.Parameter(accumulatorType, "a");
            Expression          abody1          = Expression.Call(decomposer, accumulateInfo1, p1, p2);
            LambdaExpression    accumulateExpr1 = Expression.Lambda(abody1, p1, p2);

            // RecursiveAccumulate: (TAccumulate, TAccumulate) => TAccumulate
            MethodInfo recursiveAccumulateInfo1 = decomposerType.GetMethod("RecursiveAccumulate");

            p2 = Expression.Parameter(accumulatorType, "e");
            Expression       rbody1 = Expression.Call(decomposer, recursiveAccumulateInfo1, p1, p2);
            LambdaExpression recursiveAccumulateExpr1 = Expression.Lambda(rbody1, p1, p2);

            // FinalReduce: TAccumulate => TResult
            MethodInfo       finalReduceInfo1 = decomposerType.GetMethod("FinalReduce");
            Expression       fbody1           = Expression.Call(decomposer, finalReduceInfo1, p1);
            LambdaExpression finalReduceExpr1 = Expression.Lambda(fbody1, p1);

            return(new DecompositionInfo(mcExpr, seedExpr1, accumulateExpr1, recursiveAccumulateExpr1, finalReduceExpr1));
        }
Beispiel #23
0
        private DryadQueryNode VisitQueryOperatorCall(QueryNodeInfo nodeInfo)
        {
            DryadQueryNode resNode = nodeInfo.queryNode;
            if (resNode != null) return resNode;

            MethodCallExpression expression = (MethodCallExpression)nodeInfo.queryExpression;
            string methodName = expression.Method.Name;

            #region LINQMETHODS
            switch (methodName)
            {
                case "Aggregate":
                case "AggregateAsQuery":
                {
                    bool isQuery = (methodName == "AggregateAsQuery");
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression funcLambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (funcLambda != null && funcLambda.Parameters.Count == 2)
                        {
                            resNode = this.VisitAggregate(nodeInfo.children[0].child,
                                                          null,
                                                          funcLambda,
                                                          null,
                                                          isQuery,
                                                          expression);
                        }
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression funcLambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        if (funcLambda != null && funcLambda.Parameters.Count == 2)
                        {
                            resNode = this.VisitAggregate(nodeInfo.children[0].child,
                                                          expression.Arguments[1],
                                                          funcLambda,
                                                          null,
                                                          isQuery,
                                                          expression);
                        }
                    }
                    else if (expression.Arguments.Count == 4)
                    {
                        LambdaExpression funcLambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        LambdaExpression resultLambda = HpcLinqExpression.GetLambda(expression.Arguments[3]);
                        if (funcLambda != null && funcLambda.Parameters.Count == 2 &&
                            resultLambda != null && resultLambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitAggregate(nodeInfo.children[0].child,
                                                          expression.Arguments[1],
                                                          funcLambda,
                                                          resultLambda,
                                                          isQuery,
                                                          expression);
                        }
                    }
                    break;
                }
                case "Select":
                case "LongSelect":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
                        {
                            resNode = this.VisitSelect(nodeInfo.children[0].child,
                                                       QueryNodeType.Select,
                                                       lambda,
                                                       null,
                                                       expression);
                        }
                    }
                    break;
                }
                case "SelectMany":
                case "LongSelectMany":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count <= 2)
                        {
                            resNode = this.VisitSelect(nodeInfo.children[0].child,
                                                       QueryNodeType.SelectMany,
                                                       lambda,
                                                       null,
                                                       expression);
                        }
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression lambda1 = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        if (lambda1 != null && (lambda1.Parameters.Count == 1 || lambda1.Parameters.Count == 2) &&
                            lambda2 != null && lambda2.Parameters.Count == 2)
                        {
                            resNode = this.VisitSelect(nodeInfo.children[0].child,
                                                       QueryNodeType.SelectMany,
                                                       lambda1,
                                                       lambda2,
                                                       expression);
                        }
                    }
                    break;
                }
                case "Join":
                case "GroupJoin":
                {
                    QueryNodeType nodeType = (methodName == "Join") ? QueryNodeType.Join : QueryNodeType.GroupJoin;
                    if (expression.Arguments.Count == 5)
                    {
                        LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        LambdaExpression lambda3 = HpcLinqExpression.GetLambda(expression.Arguments[3]);
                        LambdaExpression lambda4 = HpcLinqExpression.GetLambda(expression.Arguments[4]);
                        if (lambda2 != null && lambda2.Parameters.Count == 1 &&
                            lambda3 != null && lambda3.Parameters.Count == 1 &&
                            lambda4 != null && lambda4.Parameters.Count == 2)
                        {
                            resNode = this.VisitJoin(nodeInfo.children[0].child,
                                                     nodeInfo.children[1].child,
                                                     nodeType,
                                                     lambda2,
                                                     lambda3,
                                                     lambda4,
                                                     null,
                                                     expression);
                        }
                    }
                    else if (expression.Arguments.Count == 6)
                    {
                        LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        LambdaExpression lambda3 = HpcLinqExpression.GetLambda(expression.Arguments[3]);
                        LambdaExpression lambda4 = HpcLinqExpression.GetLambda(expression.Arguments[4]);
                        if (lambda2 != null && lambda2.Parameters.Count == 1 &&
                            lambda3 != null && lambda3.Parameters.Count == 1 &&
                            lambda4 != null && lambda4.Parameters.Count == 2)
                        {
                            resNode = this.VisitJoin(nodeInfo.children[0].child,
                                                     nodeInfo.children[1].child,
                                                     nodeType,
                                                     lambda2,
                                                     lambda3,
                                                     lambda4,
                                                     expression.Arguments[5],
                                                     expression);
                        }
                    }
                    break;
                }
                case "OfType":
                {
                    if (expression.Arguments.Count == 1)
                    {
                        Type ofType = expression.Method.GetGenericArguments()[0];
                        resNode = this.VisitOfType(nodeInfo.children[0].child, ofType, expression);
                    }
                    break;
                }
                case "Where":
                case "LongWhere":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
                        {
                            resNode = this.VisitWhere(nodeInfo.children[0].child,
                                                      lambda,
                                                      expression);
                        }
                    }
                    break;
                }
                case "First":
                case "FirstOrDefault":
                case "FirstAsQuery":
                {
                    AggregateOpType aggType = (methodName == "FirstOrDefault") ? AggregateOpType.FirstOrDefault : AggregateOpType.First;
                    bool isQuery = (methodName == "FirstAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitFirst(nodeInfo.children[0].child, null, aggType, isQuery, expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitFirst(nodeInfo.children[0].child,
                                                      lambda,
                                                      aggType,
                                                      isQuery,
                                                      expression);
                        }
                    }
                    break;
                }
                case "Single":
                case "SingleOrDefault":
                case "SingleAsQuery":
                {
                    AggregateOpType aggType = (methodName == "SingleOrDefault") ? AggregateOpType.SingleOrDefault : AggregateOpType.Single;
                    bool isQuery = (methodName == "SingleAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitFirst(nodeInfo.children[0].child, null, aggType, isQuery, expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitFirst(nodeInfo.children[0].child,
                                                      lambda,
                                                      aggType,
                                                      isQuery,
                                                      expression);
                        }
                    }
                    break;
                }
                case "Last":
                case "LastOrDefault":
                case "LastAsQuery":
                {
                    AggregateOpType aggType = (methodName == "LastOrDefault") ? AggregateOpType.LastOrDefault : AggregateOpType.Last;
                    bool isQuery = (methodName == "LastAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitFirst(nodeInfo.children[0].child, null, aggType, isQuery, expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitFirst(nodeInfo.children[0].child,
                                                      lambda,
                                                      aggType,
                                                      isQuery,
                                                      expression);
                        }
                    }
                    break;
                }
                case "Distinct":
                {
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitDistinct(nodeInfo.children[0].child, null, expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitDistinct(nodeInfo.children[0].child,
                                                     expression.Arguments[1],
                                                     expression);
                    }
                    break;
                }
                case "DefaultIfEmpty":
                {
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitDefaultIfEmpty(nodeInfo.children[0].child,
                                                           null,
                                                           expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitDefaultIfEmpty(nodeInfo.children[0].child,
                                                           expression.Arguments[1],
                                                           expression);
                    }
                    break;
                }
                case "Concat":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitConcat(nodeInfo, expression);
                    }
                    break;
                }
                case "Union":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitSetOperation(nodeInfo.children[0].child,
                                                         nodeInfo.children[1].child,
                                                         QueryNodeType.Union,
                                                         null,
                                                         expression);
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        resNode = this.VisitSetOperation(nodeInfo.children[0].child,
                                                         nodeInfo.children[1].child,
                                                         QueryNodeType.Union,
                                                         expression.Arguments[2],
                                                         expression);
                    }
                    break;
                }
                case "Intersect":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitSetOperation(nodeInfo.children[0].child,
                                                         nodeInfo.children[1].child,
                                                         QueryNodeType.Intersect,
                                                         null,
                                                         expression);
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        resNode = this.VisitSetOperation(nodeInfo.children[0].child,
                                                         nodeInfo.children[1].child,
                                                         QueryNodeType.Intersect,
                                                         expression.Arguments[2],
                                                         expression);
                    }
                    break;
                }
                case "Except":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitSetOperation(nodeInfo.children[0].child,
                                                         nodeInfo.children[1].child,
                                                         QueryNodeType.Except,
                                                         null,
                                                         expression);
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        resNode = this.VisitSetOperation(nodeInfo.children[0].child,
                                                         nodeInfo.children[1].child,
                                                         QueryNodeType.Except,
                                                         expression.Arguments[2],
                                                         expression);
                    }
                    break;
                }
            case "Any":
            case "AnyAsQuery":
                {
                    bool isQuery = (methodName == "AnyAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        Type type = expression.Method.GetGenericArguments()[0];
                        ParameterExpression param = Expression.Parameter(type, "x");
                        Type delegateType = typeof(Func<,>).MakeGenericType(type, typeof(bool));
                        Expression body = Expression.Constant(true);
                        LambdaExpression lambda = Expression.Lambda(delegateType, body, param);
                        resNode = this.VisitQuantifier(nodeInfo.children[0].child,
                                                       lambda,
                                                       AggregateOpType.Any,
                                                       isQuery,
                                                       expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitQuantifier(nodeInfo.children[0].child,
                                                           lambda,
                                                           AggregateOpType.Any,
                                                           isQuery,
                                                           expression);
                        }
                    }
                    break;
                }
                case "All":
                case "AllAsQuery":
                {
                    bool isQuery = (methodName == "AllAsQuery");
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitQuantifier(nodeInfo.children[0].child,
                                                           lambda,
                                                           AggregateOpType.All,
                                                           isQuery,
                                                           expression);
                        }
                    }
                    break;
                }
                case "Count":
                case "CountAsQuery":
                {
                    bool isQuery = (methodName == "CountAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                           null,
                                                           AggregateOpType.Count,
                                                           isQuery,
                                                           expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                               lambda,
                                                               AggregateOpType.Count,
                                                               isQuery,
                                                               expression);
                        }
                    }
                    break;
                }
                case "LongCount":
                case "LongCountAsQuery":
                {
                    bool isQuery = (methodName == "LongCountAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                           null,
                                                           AggregateOpType.LongCount,
                                                           isQuery,
                                                           expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                               lambda,
                                                               AggregateOpType.LongCount,
                                                               isQuery,
                                                               expression);
                        }
                    }
                    break;
                }
                case "Sum":
                case "SumAsQuery":
                {
                    bool isQuery = (methodName == "SumAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                           null,
                                                           AggregateOpType.Sum,
                                                           isQuery,
                                                           expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                               lambda,
                                                               AggregateOpType.Sum,
                                                               isQuery,
                                                               expression);
                        }
                    }
                    break;
                }
                case "Min":
                case "MinAsQuery":
                {
                    bool isQuery = (methodName == "MinAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                           null,
                                                           AggregateOpType.Min,
                                                           isQuery,
                                                           expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                               lambda,
                                                               AggregateOpType.Min,
                                                               isQuery,
                                                               expression);
                        }
                    }
                    break;
                }
                case "Max":
                case "MaxAsQuery":
                {
                    bool isQuery = (methodName == "MaxAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                           null,
                                                           AggregateOpType.Max,
                                                           isQuery,
                                                           expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                               lambda,
                                                               AggregateOpType.Max,
                                                               isQuery,
                                                               expression);
                        }
                    }
                    break;
                }
                case "Average":
                case "AverageAsQuery":
                {
                    bool isQuery = (methodName == "AverageAsQuery");
                    if (expression.Arguments.Count == 1)
                    {
                        resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                           null,
                                                           AggregateOpType.Average,
                                                           isQuery,
                                                           expression);
                    }
                    else if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
                                                               lambda,
                                                               AggregateOpType.Average,
                                                               isQuery,
                                                               expression);
                        }
                    }
                    break;
                }
                case "GroupBy":
                {
                    // groupby can take 2, 3, 4, or 5 arguments.
                    if (expression.Arguments.Count == 2)
                    {
                        //Supplied arguments are as follows:(source, key selector)
                        LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (keySelExpr != null && keySelExpr.Parameters.Count == 1)
                        {
                            resNode = this.VisitGroupBy(nodeInfo.children[0].child,
                                                        keySelExpr,
                                                        null,
                                                        null,
                                                        null,
                                                        expression);
                        }
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        //Supplied arguments are as follows:(source, key selector, element selector/result selector/comparer)
                        LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        if (keySelExpr != null && lambda2 == null)
                        {
                            resNode = this.VisitGroupBy(nodeInfo.children[0].child,
                                                        keySelExpr,
                                                        null,
                                                        null,
                                                        expression.Arguments[2],
                                                        expression);
                        }
                        else if (keySelExpr != null && keySelExpr.Parameters.Count == 1 && lambda2 != null)
                        {
                            LambdaExpression elemSelExpr = null;
                            LambdaExpression resSelExpr = null;
                            if (lambda2.Parameters.Count == 1)
                            {
                                elemSelExpr = lambda2;
                            }
                            else if (lambda2.Parameters.Count == 2)
                            {
                                resSelExpr = lambda2;
                            }
                            resNode = this.VisitGroupBy(nodeInfo.children[0].child,
                                                        keySelExpr,
                                                        elemSelExpr,
                                                        resSelExpr,
                                                        null,
                                                        expression);
                        }
                    }
                    else if (expression.Arguments.Count == 4)
                    {
                        //Argument-0 is source and Argument-1 is key selector expression
                        LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        LambdaExpression lambda3 = HpcLinqExpression.GetLambda(expression.Arguments[3]);
                        if (keySelExpr != null && keySelExpr.Parameters.Count == 1 && lambda3 == null)
                        {
                            //Argument-2 can be either result selector, element selector and argument-3 is comparer
                            LambdaExpression elemSelExpr = null;
                            LambdaExpression resSelExpr = null;
                            if (lambda2.Parameters.Count == 1)
                            {
                                elemSelExpr = lambda2;
                            }
                            else if (lambda2.Parameters.Count == 2)
                            {
                                resSelExpr = lambda2;
                            }
                            resNode = this.VisitGroupBy(nodeInfo.children[0].child,
                                                        keySelExpr,
                                                        elemSelExpr,
                                                        resSelExpr,
                                                        expression.Arguments[3],
                                                        expression);
                        }
                        else if (keySelExpr != null && keySelExpr.Parameters.Count == 1 &&
                                 lambda2 != null && lambda2.Parameters.Count == 1 &&
                                 lambda3 != null && lambda3.Parameters.Count == 2)
                        {
                            //Argument-2 is element selector and argument-3 is result selector
                            resNode = this.VisitGroupBy(nodeInfo.children[0].child,
                                                        keySelExpr,
                                                        lambda2,
                                                        lambda3,
                                                        null,
                                                        expression);
                        }
                    }
                    else if (expression.Arguments.Count == 5)
                    {
                        //Supplied arguments are as follows:(source, key selector, element selector, result selector, comparer)
                        LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        LambdaExpression elemSelExpr = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        LambdaExpression resSelExpr = HpcLinqExpression.GetLambda(expression.Arguments[3]);
                        if (keySelExpr != null && keySelExpr.Parameters.Count == 1 &&
                            elemSelExpr != null && elemSelExpr.Parameters.Count == 1 &&
                            resSelExpr != null && resSelExpr.Parameters.Count == 2)
                        {
                            resNode = this.VisitGroupBy(nodeInfo.children[0].child,
                                                        keySelExpr,
                                                        elemSelExpr,
                                                        resSelExpr,
                                                        expression.Arguments[4],
                                                        expression);
                        }
                    }
                    break;
                }
                case "OrderBy":
                case "OrderByDescending":
                {
                    bool isDescending = (methodName == "OrderByDescending");
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitOrderBy(nodeInfo.children[0].child,
                                                        lambda,
                                                        null,
                                                        isDescending,
                                                        expression);
                        }
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitOrderBy(nodeInfo.children[0].child,
                                                        lambda,
                                                        expression.Arguments[2],
                                                        isDescending,
                                                        expression);
                        }
                    }
                    break;
                }
                case "ThenBy":
                case "ThenByDescending":
                {
                    bool isDescending = (methodName == "ThenByDescending");
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitThenBy(nodeInfo.children[0].child,
                                                       lambda,
                                                       isDescending,
                                                       expression);
                        }
                    }
                    break;
                }
                case "ElementAt":
                case "ElementAtOrDefault":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitElementAt(methodName,
                                                      nodeInfo.children[0].child,
                                                      expression.Arguments[1],
                                                      expression);
                    }
                    break;
                }
                case "Take":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitPartitionOp(methodName,
                                                        nodeInfo.children[0].child,
                                                        QueryNodeType.Take,
                                                        expression.Arguments[1],
                                                        expression);
                    }
                    break;
                }
                case "TakeWhile":
                case "LongTakeWhile":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null &&
                            (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
                        {
                            resNode = this.VisitPartitionOp("TakeWhile",
                                                            nodeInfo.children[0].child,
                                                            QueryNodeType.TakeWhile,
                                                            lambda,
                                                            expression);
                        }
                    }
                    break;
                }
                case "Skip":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitPartitionOp(methodName,
                                                        nodeInfo.children[0].child,
                                                        QueryNodeType.Skip,
                                                        expression.Arguments[1],
                                                        expression);
                    }
                    break;
                }
                case "SkipWhile":
                case "LongSkipWhile":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null &&
                            (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
                        {
                            resNode = this.VisitPartitionOp(methodName,
                                                            nodeInfo.children[0].child,
                                                            QueryNodeType.SkipWhile,
                                                            lambda,
                                                            expression);
                        }
                    }
                    break;
                }
                case "Contains":
                case "ContainsAsQuery":
                {
                    bool isQuery = (methodName == "ContainsAsQuery");
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitContains(nodeInfo.children[0].child,
                                                     expression.Arguments[1],
                                                     null,
                                                     isQuery,
                                                     expression);
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        resNode = this.VisitContains(nodeInfo.children[0].child,
                                                     expression.Arguments[1],
                                                     expression.Arguments[2],
                                                     isQuery,
                                                     expression);
                    }
                    break;
                }
                case "Reverse":
                {
                    resNode = this.VisitReverse(nodeInfo.children[0].child, expression);
                    break;
                }
                case "SequenceEqual":
                case "SequenceEqualAsQuery":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitSequenceEqual(nodeInfo.children[0].child,
                                                          nodeInfo.children[1].child,
                                                          null,
                                                          expression);
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        resNode = this.VisitSequenceEqual(nodeInfo.children[0].child,
                                                          nodeInfo.children[1].child,
                                                          expression.Arguments[2],
                                                          expression);
                    }
                    break;
                }
                case "Zip":
                {
                    if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        if (lambda != null && lambda.Parameters.Count == 2)
                        {
                            resNode = this.VisitZip(nodeInfo.children[0].child,
                                                    nodeInfo.children[1].child,
                                                    lambda,
                                                    expression);
                        }
                    }
                    break;
                }
                case "HashPartition":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitHashPartition(nodeInfo.children[0].child,
                                                              lambda,
                                                              null,
                                                              null,
                                                              expression);
                        }
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            if (expression.Arguments[2].Type == typeof(int))
                            {
                                resNode = this.VisitHashPartition(nodeInfo.children[0].child,
                                                                  lambda,
                                                                  null,
                                                                  expression.Arguments[2],
                                                                  expression);
                            }
                            else
                            {
                                resNode = this.VisitHashPartition(nodeInfo.children[0].child,
                                                                  lambda,
                                                                  expression.Arguments[2],
                                                                  null,
                                                                  expression);
                            }
                        }
                    }
                    else if (expression.Arguments.Count == 4)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitHashPartition(nodeInfo.children[0].child,
                                                              lambda,
                                                              expression.Arguments[2],
                                                              expression.Arguments[3],
                                                              expression);
                        }
                    }
                    break;
                }
                case "RangePartition":
                {
                    //overloads:
                    //
                    //  2-param:
                    //    (source, keySelector)
                    //
                    //  3-param:
                    //    (source, keySelector, pcount)
                    //    (source, keySelector, isDescending)
                    //    (source, keySelector, rangeSeparators)
                    //
                    //  4-param:
                    //    (source, keySelector, isDescending, pcount)
                    //    (source, keySelector, keyComparer, isDescending)
                    //    (source, keySelector, rangeSeparators, keyComparer)
                    //
                    //  5-param:
                    //    (source, keySelector, keyComparer, isDescending, pcount)
                    //    (source, keySelector, rangeSeparators, keyComparer, isDescending)
                    if (expression.Arguments.Count == 2)
                    {
                        // Case: (source, keySelector)
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        resNode = this.VisitRangePartition(nodeInfo.children[0].child,
                                                           lambda,
                                                           null,
                                                           null,
                                                           null,
                                                           null,
                                                           expression);
                    }

                    else if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            if (expression.Arguments[2].Type == typeof(int))
                            {
                                // Case: (source, keySelector, pcount)
                                resNode = this.VisitRangePartition(nodeInfo.children[0].child,
                                                                   lambda,
                                                                   null,
                                                                   null,
                                                                   null,
                                                                   expression.Arguments[2],
                                                                   expression);
                            }
                            if (expression.Arguments[2].Type == typeof(bool))
                            {
                                // Case: (source, keySelector, isDescending)
                                resNode = this.VisitRangePartition(nodeInfo.children[0].child,
                                                                   lambda,
                                                                   null,
                                                                   null,
                                                                   expression.Arguments[2],
                                                                   null,
                                                                   expression);
                            }
                            else if (expression.Arguments[2].Type.IsArray)
                            {
                                // Case: RangePartition(keySelector, TKey[] keys)
                                resNode = this.VisitRangePartition(nodeInfo.children[0].child,
                                                                   lambda,
                                                                   expression.Arguments[2],
                                                                   null,
                                                                   null,
                                                                   null,
                                                                   expression);
                            }
                        }
                    }
                    else if (expression.Arguments.Count == 4)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            if (expression.Arguments[2].Type == typeof(bool))
                            {
                                //case: (source, keySelector, isDescending, pcount)
                                resNode = this.VisitRangePartition(nodeInfo.children[0].child,
                                                                   lambda,
                                                                   null,
                                                                   null,
                                                                   expression.Arguments[2],
                                                                   expression.Arguments[3],
                                                                   expression);
                            }
                            else if (expression.Arguments[3].Type == typeof(bool))
                            {
                                //case: (source, keySelector, keyComparer, isDescending)
                                resNode = this.VisitRangePartition(nodeInfo.children[0].child,
                                                                   lambda,
                                                                   null,
                                                                   expression.Arguments[2],
                                                                   expression.Arguments[3],
                                                                   null,
                                                                   expression);
                            }
                            else if (expression.Arguments[2].Type.IsArray)
                            {
                                //case: (source, keySelector, rangeSeparators, keyComparer)

                                resNode = this.VisitRangePartition(nodeInfo.children[0].child,
                                                                   lambda,
                                                                   expression.Arguments[2],
                                                                   expression.Arguments[3],
                                                                   null,
                                                                   null,
                                                                   expression);
                            }
                        }
                    }
                    else if (expression.Arguments.Count == 5)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            if (expression.Arguments[3].Type == typeof(bool))
                            {
                                // case: (source, keySelector, keyComparer, isDescending, pcount)
                                resNode = this.VisitRangePartition(nodeInfo.children[0].child,
                                                                  lambda,
                                                                  null,
                                                                  expression.Arguments[2],
                                                                  expression.Arguments[3],
                                                                  expression.Arguments[4],
                                                                  expression);
                            }
                            else if (expression.Arguments[4].Type == typeof(bool))
                            {
                                // case: (source, keySelector, rangeSeparators, keyComparer, isDescending)
                                resNode = this.VisitRangePartition(nodeInfo.children[0].child,
                                                                   lambda,
                                                                   expression.Arguments[2],
                                                                   expression.Arguments[3],
                                                                   expression.Arguments[4],
                                                                   null,
                                                                   expression);
                            }

                        }
                    }
                    break;
                }
                case "Apply":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null &&
                            (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
                        {
                            resNode = this.VisitApply(nodeInfo.children[0].child,
                                                      null,
                                                      lambda,
                                                      false,
                                                      false,
                                                      expression);
                        }
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        if (lambda != null)
                        {
                            if (lambda.Parameters.Count == 2)
                            {
                                resNode = this.VisitApply(nodeInfo.children[0].child,
                                                          nodeInfo.children[1].child,
                                                          lambda,
                                                          false,
                                                          false,
                                                          expression);
                            }
                            else
                            {
                                // Apply with multiple sources of the same type
                                resNode = this.VisitMultiApply(nodeInfo, lambda, false, false, expression);
                            }
                        }
                    }
                    break;
                }
                case "ApplyPerPartition":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null &&
                            (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
                        {
                            resNode = this.VisitApply(nodeInfo.children[0].child,
                                                      null,
                                                      lambda,
                                                      true,
                                                      false,
                                                      expression);
                        }
                    }
                    else if (expression.Arguments.Count == 4)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
                        ExpressionSimplifier<bool> evaluator = new ExpressionSimplifier<bool>();
                        bool isFirstOnly = evaluator.Eval(expression.Arguments[3]);
                        if (lambda != null)
                        {
                            if (lambda.Parameters.Count == 2)
                            {
                                resNode = this.VisitApply(nodeInfo.children[0].child,
                                                          nodeInfo.children[1].child,
                                                          lambda,
                                                          true,
                                                          isFirstOnly,
                                                          expression);
                            }
                            else
                            {
                                // Apply with multiple sources of the same type
                                resNode = this.VisitMultiApply(nodeInfo, lambda, true, isFirstOnly, expression);
                            }
                        }
                    }
                    break;
                }
                case "Fork":
                {
                    if (expression.Arguments.Count == 2) // ForkSelect and ForkApply
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitFork(nodeInfo.children[0].child,
                                                     lambda,
                                                     null,
                                                     expression);
                        }
                    }
                    else if (expression.Arguments.Count == 3) // ForkByKey
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitFork(nodeInfo.children[0].child,
                                                     lambda,
                                                     expression.Arguments[2],
                                                     expression);
                        }
                    }
                    break;
                }
                case "ForkChoose":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        resNode = this.VisitForkChoose(nodeInfo.children[0].child,
                                                       expression.Arguments[1],
                                                       expression);
                    }
                    break;
                }
                case "AssumeHashPartition":
                {
                    if (expression.Arguments.Count == 2)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitAssumeHashPartition(nodeInfo.children[0].child,
                                                                    lambda,
                                                                    null,
                                                                    null,
                                                                    expression);
                        }
                    }
                    else if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitAssumeHashPartition(nodeInfo.children[0].child,
                                                                    lambda,
                                                                    null,
                                                                    expression.Arguments[2],
                                                                    expression);
                        }
                    }
                    break;
                }
                case "AssumeRangePartition":
                {
                    if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            if (expression.Arguments[2].Type.IsArray)
                            {
                                resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
                                                                         lambda,
                                                                         expression.Arguments[2],
                                                                         null,
                                                                         null,
                                                                         expression);
                            }
                            else
                            {
                                resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
                                                                         lambda,
                                                                         null,
                                                                         null,
                                                                         expression.Arguments[2],
                                                                         expression);
                            }
                        }
                    }
                    else if (expression.Arguments.Count == 4)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            if (expression.Arguments[2].Type.IsArray)
                            {
                                resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
                                                                         lambda,
                                                                         expression.Arguments[2],
                                                                         expression.Arguments[3],
                                                                         null,
                                                                         expression);
                            }
                            else
                            {
                                resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
                                                                         lambda,
                                                                         null,
                                                                         expression.Arguments[2],
                                                                         expression.Arguments[3],
                                                                         expression);
                            }
                        }
                    }
                    break;
                }
                case "AssumeOrderBy":
                {
                    if (expression.Arguments.Count == 3)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitAssumeOrderBy(nodeInfo.children[0].child,
                                                              lambda,
                                                              null,
                                                              expression.Arguments[2],
                                                              expression);
                        }
                    }
                    else if (expression.Arguments.Count == 4)
                    {
                        LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                        if (lambda != null && lambda.Parameters.Count == 1)
                        {
                            resNode = this.VisitAssumeOrderBy(nodeInfo.children[0].child,
                                                              lambda,
                                                              expression.Arguments[2],
                                                              expression.Arguments[3],
                                                              expression);
                        }
                    }
                    break;
                }
                case "SlidingWindow":
                {
                    LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                    if (lambda != null && lambda.Parameters.Count == 1)
                    {
                        resNode = this.VisitSlidingWindow(nodeInfo.children[0].child,
                                                          lambda,
                                                          expression.Arguments[2],
                                                          expression);
                    }
                    break;
                }
                case "SelectWithPartitionIndex":
                case "ApplyWithPartitionIndex":
                {
                    LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
                    if (lambda != null && lambda.Parameters.Count == 1)
                    {
                        resNode = this.VisitApplyWithPartitionIndex(nodeInfo.children[0].child,
                                                                    lambda,
                                                                    expression);
                    }
                    break;
                }
                case ReflectedNames.DryadLinqIQueryable_ToDscWorker: // was case "ToPartitionedTableLazy":
                {
                    //SHOULD NOT VISIT.. The DryadQueryGen ctors should be interrogating ToDsc nodes directly.
                    //Later if we do allow ToDsc in the middle of query chain, then we need to either
                    //    1. update the source node with an outputeTableUri
                    // OR 2. create an actual node and handle it later on  (tee, etc)

                    throw DryadLinqException.Create(HpcLinqErrorCode.ToDscUsedIncorrectly,
                                                  String.Format(SR.ToDscUsedIncorrectly),
                                                  expression);
                }
                case ReflectedNames.DryadLinqIQueryable_ToHdfsWorker: // was case "ToPartitionedTableLazy":
                {
                    //SHOULD NOT VISIT.. The DryadQueryGen ctors should be interrogating ToDsc nodes directly.
                    //Later if we do allow ToDsc in the middle of query chain, then we need to either
                    //    1. update the source node with an outputeTableUri
                    // OR 2. create an actual node and handle it later on  (tee, etc)

                    throw DryadLinqException.Create(HpcLinqErrorCode.ToHdfsUsedIncorrectly, String.Format(SR.ToHdfsUsedIncorrectly), expression);
                }
            }
            #endregion

            if (resNode == null)
            {
                throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
                                              String.Format(SR.OperatorNotSupported, methodName),
                                              expression);
            }
            resNode.IsForked = resNode.IsForked || nodeInfo.IsForked;
            nodeInfo.queryNode = resNode;
            return resNode;
        }
Beispiel #24
0
        internal DLinqGroupByNode(string opName,
                                  LambdaExpression keySelectExpr,
                                  LambdaExpression elemSelectExpr,
                                  LambdaExpression resSelectExpr,
                                  LambdaExpression seedExpr,
                                  LambdaExpression accumulateExpr,
                                  LambdaExpression recursiveAccumulatorExpr,
                                  Expression comparerExpr,
                                  bool isPartial,
                                  Expression queryExpr,
                                  DLinqQueryNode child)
            : base(QueryNodeType.GroupBy, child.QueryGen, queryExpr, child)
        {
            Debug.Assert(opName == "GroupBy" || opName == "OrderedGroupBy");
            this.m_keySelectExpr = keySelectExpr;
            this.m_elemSelectExpr = elemSelectExpr;
            this.m_resSelectExpr = resSelectExpr;
            this.m_seedExpr = seedExpr;
            this.m_accumulatorExpr = accumulateExpr;
            this.m_recursiveAccumulatorExpr = recursiveAccumulatorExpr;
            this.m_comparerExpr = comparerExpr;
            this.m_isPartial = isPartial;
            this.m_opName = opName;

            this.m_comparer = null;
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                this.m_comparer = evaluator.Eval(comparerExpr);
                this.m_comparerIdx = DryadLinqObjectStore.Put(this.m_comparer);
            }

            this.m_partitionCount = child.OutputDataSetInfo.partitionInfo.Count;
            this.m_outputDataSetInfo = this.ComputeOutputDataSetInfo(isPartial);

            this.m_dynamicManager = this.InferDynamicManager();
        }
Beispiel #25
0
        private DryadQueryNode VisitSlidingWindow(QueryNodeInfo source,
                                                  LambdaExpression procLambda,
                                                  Expression windowSizeExpr,
                                                  Expression queryExpr)
        {
            // var windows = source.Apply(s => HpcLinqHelper.Last(s, windowSize));
            // var slided = windows.Apply(s => HpcLinqHelper.Slide(s)).HashPartition(x => x.Index);
            // slided.Apply(source, (x, y) => HpcLinqHelper.ProcessWindows(x, y, procFunc, windowSize));
            DryadQueryNode child = this.Visit(source);
            if (child.IsDynamic)
            {
                throw new DryadLinqException("SlidingWindow is only supported for static partition count");
            }

            ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
            Expression windowSize = Expression.Constant(evaluator.Eval(windowSizeExpr), typeof(int));

            child.IsForked = true;

            // Apply node for s => Last(s, windowSize)
            Type paramType = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
            ParameterExpression param = Expression.Parameter(paramType, HpcLinqCodeGen.MakeUniqueName("s"));
            MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("Last");
            minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
            Expression body = Expression.Call(minfo, param, windowSize);
            Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
            LambdaExpression procFunc = Expression.Lambda(funcType, body, param);
            DryadQueryNode lastNode = new DryadApplyNode(procFunc, queryExpr, child);
            lastNode = new DryadMergeNode(true, true, queryExpr, lastNode);

            // Apply node for s => Slide(s)
            param = Expression.Parameter(body.Type, HpcLinqCodeGen.MakeUniqueName("s"));
            minfo = typeof(HpcLinqHelper).GetMethod("Slide");
            minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
            body = Expression.Call(minfo, param);
            funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
            procFunc = Expression.Lambda(funcType, body, param);
            DryadQueryNode slideNode = new DryadApplyNode(procFunc, queryExpr, lastNode);

            // Hash partition to distribute from partition i to i+1
            int pcount = child.OutputPartition.Count;
            param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
            Expression keySelectBody = Expression.Property(param, "Index");
            funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
            LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
            DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr,
                                                                  null,
                                                                  pcount,
                                                                  queryExpr,
                                                                  slideNode);

            // Apply node for (x, y) => ProcessWindows(x, y, proclambda, windowSize)
            Type paramType1 = typeof(IEnumerable<>).MakeGenericType(body.Type);
            ParameterExpression param1 = Expression.Parameter(paramType1, HpcLinqCodeGen.MakeUniqueName("x"));
            Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
            ParameterExpression param2 = Expression.Parameter(paramType2, HpcLinqCodeGen.MakeUniqueName("y"));
            minfo = typeof(HpcLinqHelper).GetMethod("ProcessWindows");
            minfo = minfo.MakeGenericMethod(child.OutputTypes[0], procLambda.Body.Type);
            body = Expression.Call(minfo, param1, param2, procLambda, windowSize);
            funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
            procFunc = Expression.Lambda(funcType, body, param1, param2);
            return new DryadApplyNode(procFunc, queryExpr, hdistNode, child);
        }
Beispiel #26
0
        internal DLinqPartitionOpNode(string opName,
                                      QueryNodeType nodeType,
                                      Expression controlExpr,
                                      bool isFirstStage,
                                      Expression queryExpr,
                                      DLinqQueryNode child)
            : base(nodeType, child.QueryGen, queryExpr, child)
        {
            this.m_controlExpression = controlExpr;
            this.m_isFirstStage = isFirstStage;
            this.m_opName = opName;
            
            this.m_count = -1;
            if (nodeType == QueryNodeType.Take || nodeType == QueryNodeType.Skip)
            {
                ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
                this.m_count = evaluator.Eval(controlExpr);
            }

            DataSetInfo childInfo = child.OutputDataSetInfo;
            if (isFirstStage)
            {
                this.m_partitionCount = child.OutputPartition.Count;
                this.m_outputDataSetInfo = new DataSetInfo(child.OutputDataSetInfo);
                this.m_dynamicManager = this.InferDynamicManager();
            }
            else
            {
                this.m_partitionCount = 1;
                this.m_outputDataSetInfo = new DataSetInfo(childInfo);
                this.m_outputDataSetInfo.partitionInfo = DataSetInfo.OnePartition;
                if (childInfo.partitionInfo.Count > 1 &&
                    (childInfo.partitionInfo.ParType != PartitionType.Range ||
                     childInfo.orderByInfo.IsOrdered))
                {
                    this.m_outputDataSetInfo.orderByInfo = DataSetInfo.NoOrderBy;
                }
                this.m_dynamicManager = DynamicManager.None;                
            }
        }
Beispiel #27
0
        // This constructor is specifically to support Materialize() calls.
        // it assumes that the Expressions all terminate with a ToDsc node.
        internal HpcLinqQueryGen(HpcLinqContext context,
                                 VertexCodeGen vertexCodeGen,
                                 Expression[] qlist)
        {
            this.m_queryExprs = new Expression[qlist.Length];
            this.m_outputTableUris = new string[qlist.Length];
            this.m_isTempOutput = new bool[qlist.Length];
            this.m_context = context;
            for (int i = 0; i < this.m_queryExprs.Length; i++)
            {
                MethodCallExpression mcExpr = (MethodCallExpression)qlist[i];
                string tableUri;
                this.m_queryExprs[i] = mcExpr.Arguments[0];

                //this block supports the scenario: q-nonToDsc
                if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_AnonymousDscPlaceholder)
                {
                    ExpressionSimplifier<string> e1 = new ExpressionSimplifier<string>();
                    tableUri = e1.Eval(mcExpr.Arguments[1]);
                    this.m_isTempOutput[i] = true;
                }

                //this block supports the scenario: q.ToDsc()
                else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToDscWorker)
                {
                    DscService dsc = context.DscService;
                    ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>();
                    string streamName = e2.Eval(mcExpr.Arguments[2]);

                    tableUri = DataPath.MakeDscStreamUri(dsc, streamName);
                    this.m_isTempOutput[i] = false;
                }

                //this block supports the scenario: q.ToHdfs()
                else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToHdfsWorker)
                {
                    string hdfsHeadNode = context.HdfsService;
                    ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>();
                    string streamName = e2.Eval(mcExpr.Arguments[2]);

                    tableUri = DataPath.MakeHdfsStreamUri(hdfsHeadNode, streamName);
                    this.m_isTempOutput[i] = false;
                }
                else {
                    throw new InvalidOperationException(); // should not occur.
                }

                this.m_outputTableUris[i] = tableUri;

            }
            this.Initialize(vertexCodeGen);
        }
Beispiel #28
0
        internal DLinqOrderByNode(LambdaExpression keySelectExpr,
                                  Expression comparerExpr,
                                  bool isDescending,
                                  Expression queryExpr,
                                  DLinqQueryNode child)
            : base(QueryNodeType.OrderBy, child.QueryGen, queryExpr, child)
        {
            this.m_keySelectExpr = keySelectExpr;
            this.m_comparerExpr = comparerExpr;
            this.m_isDescending = isDescending;
            this.m_opName = "Sort";

            this.m_comparer = null;
            this.m_comparerIdx = -1;
            if (comparerExpr != null)
            {
                ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
                this.m_comparer = evaluator.Eval(comparerExpr);
                this.m_comparerIdx = DryadLinqObjectStore.Put(this.m_comparer);
            }

            this.m_partitionCount = child.OutputPartition.Count;

            this.m_outputDataSetInfo = new DataSetInfo(child.OutputDataSetInfo);
            Type[] typeArgs = this.KeySelectExpression.Type.GetGenericArguments();
            this.m_outputDataSetInfo.orderByInfo = OrderByInfo.Create(this.KeySelectExpression,
                                                                      this.m_comparer,
                                                                      this.m_isDescending,
                                                                      typeArgs[1]);

            this.m_dynamicManager = this.InferDynamicManager();
        }