Пример #1
0
 internal virtual string Visit(DryadOutputNode node,
                               CodeMemberMethod vertexMethod,
                               string[] readerNames,
                               string[] writerNames)
 {
     return node.AddVertexCode(vertexMethod, readerNames, writerNames);
 }
Пример #2
0
        // Phase 1 of the query optimization
        internal void GenerateQueryPlanPhase1()
        {
            if (this.m_queryPlan1 != null) return;

            // Apply some simple rewrite rules
            SimpleRewriter rewriter = new SimpleRewriter(this.m_exprNodeInfoMap.Values.ToList());
            rewriter.Rewrite();

            // Generate the query plan of phase1
            var referencedNodes = this.m_referencedQueryMap.Values;
            this.m_queryPlan1 = new DryadQueryNode[this.m_queryExprs.Length + referencedNodes.Count];
            for (int i = 0; i < this.m_queryExprs.Length; i++)
            {
                this.m_queryPlan1[i] = this.Visit(this.m_queryNodeInfos[i].children[0].child);
            }
            int idx = this.m_queryExprs.Length;
            foreach (QueryNodeInfo nodeInfo in referencedNodes)
            {
                // Add a Tee'd Merge
                this.m_queryPlan1[idx] = this.Visit(nodeInfo.children[0].child);
                DryadQueryNode mergeNode = new DryadMergeNode(true, false, nodeInfo.queryExpression,
                                                              this.m_queryPlan1[idx]);
                this.m_queryPlan1[idx] =  new DryadTeeNode(mergeNode.OutputTypes[0], true,
                                                           mergeNode.QueryExpression, mergeNode);
                nodeInfo.queryNode = this.m_queryPlan1[idx];
                idx++;
            }

            // Finally, add the output nodes.
            Dictionary<DryadQueryNode, int> forkCounts = new Dictionary<DryadQueryNode, int>();
            for (int i = 0; i < this.m_queryExprs.Length; i++)
            {
                DryadQueryNode queryNode = this.m_queryPlan1[i];
                int cnt;
                if (!forkCounts.TryGetValue(queryNode, out cnt))
                {
                    cnt = queryNode.Parents.Count;
                }
                forkCounts[queryNode] = cnt + 1;
            }

            for (int i = 0; i < this.m_queryExprs.Length; i++)
            {
                HpcClientSideLog.Add("Query " + i + " Output: " + this.m_outputDatapaths[i]);

                DryadQueryNode queryNode = this.m_queryPlan1[i];
                if (TypeSystem.IsAnonymousType(queryNode.OutputTypes[0]))
                {
                    throw new DryadLinqException(HpcLinqErrorCode.OutputTypeCannotBeAnonymous,
                                               SR.OutputTypeCannotBeAnonymous);
                }

                // Add dummy Apply to make Dryad happy (it doesn't like to hook inputs straight to outputs)
                if ((queryNode is DryadInputNode) || (forkCounts[queryNode] > 1))
                {
                    // Add a dummy Apply
                    Type paramType = typeof(IEnumerable<>).MakeGenericType(queryNode.OutputTypes[0]);
                    ParameterExpression param = Expression.Parameter(paramType, "x");
                    Type type = typeof(Func<,>).MakeGenericType(paramType, paramType);
                    LambdaExpression applyExpr = Expression.Lambda(type, param, param);
                    DryadQueryNode applyNode = new DryadApplyNode(applyExpr, this.m_queryExprs[i], queryNode);
                    applyNode.OutputDataSetInfo = queryNode.OutputDataSetInfo;
                    queryNode = applyNode;
                }

                if (queryNode is DryadConcatNode)
                {
                    // Again, we add dummy Apply in certain cases to make Dryad happy
                    ((DryadConcatNode)queryNode).FixInputs();
                }

                // Add the output node
                DscCompressionScheme outputScheme = this.m_context.Configuration.OutputDataCompressionScheme;
                DryadOutputNode outputNode = new DryadOutputNode(this.m_context,
                                                                 this.m_outputDatapaths[i],
                                                                 this.m_isTempOutput[i],
                                                                 outputScheme,
                                                                 this.m_queryExprs[i],
                                                                 queryNode);

                this.m_queryPlan1[i] = outputNode;

                if (this.m_outputUriMap.ContainsKey(this.m_outputDatapaths[i].ToLower()))
                {
                    throw new DryadLinqException(HpcLinqErrorCode.MultipleOutputsWithSameDscUri,
                                               String.Format(SR.MultipleOutputsWithSameDscUri, this.m_outputDatapaths[i]));
                }

                this.m_outputUriMap.Add(this.m_outputDatapaths[i].ToLower(), outputNode);
                this.m_outputTypes[i] = this.m_queryPlan1[i].OutputTypes[0];

                // Remove useless Tees to make Dryad happy
                if ((queryNode is DryadTeeNode) && (forkCounts[queryNode] == 1))
                {
                    DryadQueryNode teeChild = queryNode.Children[0];
                    teeChild.UpdateParent(queryNode, outputNode);
                    outputNode.UpdateChildren(queryNode, teeChild);
                }
            }
        }
Пример #3
0
        // create DryadLinqMetaData from a query OutputNode
        internal static DryadLinqMetaData FromOutputNode(HpcLinqContext context, DryadOutputNode node)
        {
            DryadLinqMetaData metaData = new DryadLinqMetaData();

            if (! (DataPath.IsDsc(node.MetaDataUri) || DataPath.IsHdfs(node.MetaDataUri)) )
            {
                throw new InvalidOperationException();
            }

            metaData.m_context = context;
            metaData.m_dscStreamName = node.MetaDataUri;
            metaData.m_elemType = node.OutputTypes[0];
            metaData.m_compressionScheme = node.OutputCompressionScheme;
            //metaData.m_version = context.ClientVersion;
            //metaData.InitializeFlags();

            //metaData.m_fp = 0UL;
            //metaData.m_dataSetInfo = node.OutputDataSetInfo;

            return metaData;
        }