// create DryadLinqMetaData from a query OutputNode internal static DryadLinqMetaData Get(DryadLinqContext context, DLinqOutputNode node) { DryadLinqMetaData metaData = new DryadLinqMetaData(); if (!DataPath.IsValidDataPath(node.OutputUri)) { throw new InvalidOperationException(); } metaData.m_context = context; metaData.m_dataSetUri = node.OutputUri; metaData.m_elemType = node.OutputTypes[0]; metaData.m_compressionScheme = node.OutputCompressionScheme; //metaData.m_version = context.ClientVersion(); //metaData.InitializeFlags(); //metaData.m_fp = 0UL; //metaData.m_dataSetInfo = node.OutputDataSetInfo; return metaData; }
internal virtual string Visit(DLinqOutputNode node, CodeMemberMethod vertexMethod, string[] readerNames, string[] writerNames) { return node.AddVertexCode(vertexMethod, readerNames, writerNames); }
// Phase 1 of the query optimization internal void GenerateQueryPlanPhase1() { if (this.m_queryPlan1 != null) return; // Apply some simple rewrite rules SimpleRewriter rewriter = new SimpleRewriter(this.m_exprNodeInfoMap.Values.ToList()); rewriter.Rewrite(); // Generate the query plan of phase1 var referencedNodes = this.m_referencedQueryMap.Values; this.m_queryPlan1 = new DLinqQueryNode[this.m_queryExprs.Length + referencedNodes.Count]; for (int i = 0; i < this.m_queryExprs.Length; i++) { this.m_queryPlan1[i] = this.Visit(this.m_queryNodeInfos[i].Children[0].Child); } int idx = this.m_queryExprs.Length; foreach (DummyQueryNodeInfo nodeInfo in referencedNodes) { if (nodeInfo.NeedsMerge) { // Add a Tee'd Merge this.m_queryPlan1[idx] = this.Visit(nodeInfo.Children[0].Child); DLinqQueryNode mergeNode = new DLinqMergeNode(true, nodeInfo.QueryExpression, this.m_queryPlan1[idx]); this.m_queryPlan1[idx] = new DLinqTeeNode(mergeNode.OutputTypes[0], true, mergeNode.QueryExpression, mergeNode); } else { this.m_queryPlan1[idx] = this.Visit(nodeInfo.Children[0].Child); } nodeInfo.QueryNode = this.m_queryPlan1[idx]; idx++; } // Finally, add the output nodes. Dictionary<DLinqQueryNode, int> forkCounts = new Dictionary<DLinqQueryNode, int>(); for (int i = 0; i < this.m_queryExprs.Length; i++) { DLinqQueryNode queryNode = this.m_queryPlan1[i]; int cnt; if (!forkCounts.TryGetValue(queryNode, out cnt)) { cnt = queryNode.Parents.Count; } forkCounts[queryNode] = cnt + 1; } for (int i = 0; i < this.m_queryExprs.Length; i++) { DryadLinqClientLog.Add("Query " + i + " Output: " + this.m_outputTableUris[i]); DLinqQueryNode queryNode = this.m_queryPlan1[i]; if (TypeSystem.IsAnonymousType(queryNode.OutputTypes[0])) { throw new DryadLinqException(DryadLinqErrorCode.OutputTypeCannotBeAnonymous, SR.OutputTypeCannotBeAnonymous); } if (this.m_serializers[i] != null) { // Add an Apply for the serializer if it is not null LambdaExpression serializer = DryadLinqExpression.GetLambda(this.m_serializers[i]); DLinqQueryNode applyNode = new DLinqApplyNode(serializer, this.m_queryExprs[i], queryNode); applyNode.OutputDataSetInfo = queryNode.OutputDataSetInfo; queryNode = applyNode; } else { // Add dummy Apply to make Dryad happy (it doesn't like to hook inputs straight to outputs) if ((queryNode is DLinqInputNode) || (forkCounts[queryNode] > 1)) { // Add a dummy Apply Type paramType = typeof(IEnumerable<>).MakeGenericType(queryNode.OutputTypes[0]); ParameterExpression param = Expression.Parameter(paramType, "x"); Type type = typeof(Func<,>).MakeGenericType(paramType, paramType); LambdaExpression applyExpr = Expression.Lambda(type, param, param); DLinqQueryNode applyNode = new DLinqApplyNode(applyExpr, this.m_queryExprs[i], queryNode); applyNode.OutputDataSetInfo = queryNode.OutputDataSetInfo; queryNode = applyNode; } if (queryNode is DLinqConcatNode) { // Again, we add dummy Apply in certain cases to make Dryad happy ((DLinqConcatNode)queryNode).FixInputs(); } } // Add the output node CompressionScheme outputScheme = this.m_context.OutputDataCompressionScheme; DLinqOutputNode outputNode = new DLinqOutputNode(this.m_context, this.m_outputTableUris[i], this.m_isTempOutput[i], outputScheme, this.m_queryExprs[i], queryNode); this.m_queryPlan1[i] = outputNode; string outputUri = this.m_outputTableUris[i].AbsoluteUri.ToLower(); if (this.m_outputUriMap.ContainsKey(outputUri)) { throw new DryadLinqException(DryadLinqErrorCode.MultipleOutputsWithSameDscUri, String.Format(SR.MultipleOutputsWithSameUri, this.m_outputTableUris[i])); } this.m_outputUriMap.Add(outputUri, outputNode); this.m_outputTypes[i] = this.m_queryPlan1[i].OutputTypes[0]; // Remove useless Tees to make Dryad happy if ((queryNode is DLinqTeeNode) && (forkCounts[queryNode] == 1)) { DLinqQueryNode teeChild = queryNode.Children[0]; teeChild.UpdateParent(queryNode, outputNode); outputNode.UpdateChildren(queryNode, teeChild); } } }