internal virtual string Visit(DryadOutputNode node, CodeMemberMethod vertexMethod, string[] readerNames, string[] writerNames) { return node.AddVertexCode(vertexMethod, readerNames, writerNames); }
// Phase 1 of the query optimization internal void GenerateQueryPlanPhase1() { if (this.m_queryPlan1 != null) return; // Apply some simple rewrite rules SimpleRewriter rewriter = new SimpleRewriter(this.m_exprNodeInfoMap.Values.ToList()); rewriter.Rewrite(); // Generate the query plan of phase1 var referencedNodes = this.m_referencedQueryMap.Values; this.m_queryPlan1 = new DryadQueryNode[this.m_queryExprs.Length + referencedNodes.Count]; for (int i = 0; i < this.m_queryExprs.Length; i++) { this.m_queryPlan1[i] = this.Visit(this.m_queryNodeInfos[i].children[0].child); } int idx = this.m_queryExprs.Length; foreach (QueryNodeInfo nodeInfo in referencedNodes) { // Add a Tee'd Merge this.m_queryPlan1[idx] = this.Visit(nodeInfo.children[0].child); DryadQueryNode mergeNode = new DryadMergeNode(true, false, nodeInfo.queryExpression, this.m_queryPlan1[idx]); this.m_queryPlan1[idx] = new DryadTeeNode(mergeNode.OutputTypes[0], true, mergeNode.QueryExpression, mergeNode); nodeInfo.queryNode = this.m_queryPlan1[idx]; idx++; } // Finally, add the output nodes. Dictionary<DryadQueryNode, int> forkCounts = new Dictionary<DryadQueryNode, int>(); for (int i = 0; i < this.m_queryExprs.Length; i++) { DryadQueryNode queryNode = this.m_queryPlan1[i]; int cnt; if (!forkCounts.TryGetValue(queryNode, out cnt)) { cnt = queryNode.Parents.Count; } forkCounts[queryNode] = cnt + 1; } for (int i = 0; i < this.m_queryExprs.Length; i++) { HpcClientSideLog.Add("Query " + i + " Output: " + this.m_outputDatapaths[i]); DryadQueryNode queryNode = this.m_queryPlan1[i]; if (TypeSystem.IsAnonymousType(queryNode.OutputTypes[0])) { throw new DryadLinqException(HpcLinqErrorCode.OutputTypeCannotBeAnonymous, SR.OutputTypeCannotBeAnonymous); } // Add dummy Apply to make Dryad happy (it doesn't like to hook inputs straight to outputs) if ((queryNode is DryadInputNode) || (forkCounts[queryNode] > 1)) { // Add a dummy Apply Type paramType = typeof(IEnumerable<>).MakeGenericType(queryNode.OutputTypes[0]); ParameterExpression param = Expression.Parameter(paramType, "x"); Type type = typeof(Func<,>).MakeGenericType(paramType, paramType); LambdaExpression applyExpr = Expression.Lambda(type, param, param); DryadQueryNode applyNode = new DryadApplyNode(applyExpr, this.m_queryExprs[i], queryNode); applyNode.OutputDataSetInfo = queryNode.OutputDataSetInfo; queryNode = applyNode; } if (queryNode is DryadConcatNode) { // Again, we add dummy Apply in certain cases to make Dryad happy ((DryadConcatNode)queryNode).FixInputs(); } // Add the output node DscCompressionScheme outputScheme = this.m_context.Configuration.OutputDataCompressionScheme; DryadOutputNode outputNode = new DryadOutputNode(this.m_context, this.m_outputDatapaths[i], this.m_isTempOutput[i], outputScheme, this.m_queryExprs[i], queryNode); this.m_queryPlan1[i] = outputNode; if (this.m_outputUriMap.ContainsKey(this.m_outputDatapaths[i].ToLower())) { throw new DryadLinqException(HpcLinqErrorCode.MultipleOutputsWithSameDscUri, String.Format(SR.MultipleOutputsWithSameDscUri, this.m_outputDatapaths[i])); } this.m_outputUriMap.Add(this.m_outputDatapaths[i].ToLower(), outputNode); this.m_outputTypes[i] = this.m_queryPlan1[i].OutputTypes[0]; // Remove useless Tees to make Dryad happy if ((queryNode is DryadTeeNode) && (forkCounts[queryNode] == 1)) { DryadQueryNode teeChild = queryNode.Children[0]; teeChild.UpdateParent(queryNode, outputNode); outputNode.UpdateChildren(queryNode, teeChild); } } }
// create DryadLinqMetaData from a query OutputNode internal static DryadLinqMetaData FromOutputNode(HpcLinqContext context, DryadOutputNode node) { DryadLinqMetaData metaData = new DryadLinqMetaData(); if (! (DataPath.IsDsc(node.MetaDataUri) || DataPath.IsHdfs(node.MetaDataUri)) ) { throw new InvalidOperationException(); } metaData.m_context = context; metaData.m_dscStreamName = node.MetaDataUri; metaData.m_elemType = node.OutputTypes[0]; metaData.m_compressionScheme = node.OutputCompressionScheme; //metaData.m_version = context.ClientVersion; //metaData.InitializeFlags(); //metaData.m_fp = 0UL; //metaData.m_dataSetInfo = node.OutputDataSetInfo; return metaData; }