// Add a new vertex method to the DryadLinq vertex class internal CodeMemberMethod AddVertexMethod(DLinqQueryNode node) { CodeMemberMethod vertexMethod = new CodeMemberMethod(); vertexMethod.Attributes = MemberAttributes.Public | MemberAttributes.Static; vertexMethod.ReturnType = new CodeTypeReference(typeof(int)); vertexMethod.Parameters.Add(new CodeParameterDeclarationExpression(typeof(string), "args")); vertexMethod.Name = MakeUniqueName(node.NodeType.ToString()); CodeTryCatchFinallyStatement tryBlock = new CodeTryCatchFinallyStatement(); string startedMsg = "DryadLinqLog.AddInfo(\"Vertex " + vertexMethod.Name + " started at {0}\", DateTime.Now.ToString(\"MM/dd/yyyy HH:mm:ss.fff\"))"; vertexMethod.Statements.Add(new CodeSnippetExpression(startedMsg)); // We need to add a call to CopyResources() vertexMethod.Statements.Add(new CodeSnippetExpression("CopyResources()")); if (StaticConfig.LaunchDebugger) { // If static config requests it, we do an unconditional Debugger.Launch() at vertex entry. // Currently this isn't used because StaticConfig.LaunchDebugger is hardcoded to false System.Console.WriteLine("Launch debugger: may block application"); CodeExpression launchExpr = new CodeSnippetExpression("System.Diagnostics.Debugger.Launch()"); vertexMethod.Statements.Add(new CodeExpressionStatement(launchExpr)); } else { // Otherwise (the default behavior), we check an environment variable to decide whether // to launch the debugger, wait for a manual attach or simply skip straigt into vertex code. CodeMethodInvokeExpression debuggerCheckExpr = new CodeMethodInvokeExpression( new CodeMethodReferenceExpression(new CodeTypeReferenceExpression(HelperClassName), DebugHelperMethodName)); vertexMethod.Statements.Add(new CodeExpressionStatement(debuggerCheckExpr)); } vertexMethod.Statements.Add(MakeVertexParamsDecl(node)); vertexMethod.Statements.Add(SetVertexParamField("VertexStageName", vertexMethod.Name)); vertexMethod.Statements.Add(SetVertexParamField("UseLargeBuffer", node.UseLargeWriteBuffer)); Int32[] portCountArray = node.InputPortCounts(); bool[] keepPortOrderArray = node.KeepInputPortOrders(); for (int i = 0; i < node.InputArity; i++) { CodeExpression setParamsExpr = new CodeMethodInvokeExpression( new CodeVariableReferenceExpression(VertexParamName), "SetInputParams", new CodePrimitiveExpression(i), new CodePrimitiveExpression(portCountArray[i]), new CodePrimitiveExpression(keepPortOrderArray[i])); vertexMethod.Statements.Add(new CodeExpressionStatement(setParamsExpr)); } // YY: We could probably do better here. for (int i = 0; i < node.GetReferencedQueries().Count; i++) { CodeExpression setParamsExpr = new CodeMethodInvokeExpression( new CodeVariableReferenceExpression(VertexParamName), "SetInputParams", new CodePrimitiveExpression(i + node.InputArity), new CodePrimitiveExpression(1), new CodePrimitiveExpression(false)); vertexMethod.Statements.Add(new CodeExpressionStatement(setParamsExpr)); } // Push the parallel-code settings into DryadLinqVertex bool multiThreading = this.m_context.EnableMultiThreadingInVertex; vertexMethod.Statements.Add(SetVertexParamField("MultiThreading", multiThreading)); vertexMethod.Statements.Add( new CodeAssignStatement( new CodeFieldReferenceExpression(DLVTypeExpr, "s_multiThreading"), new CodePrimitiveExpression(multiThreading))); vertexMethod.Statements.Add(MakeVertexEnvDecl(node)); Type[] outputTypes = node.OutputTypes; string[] writerNames = new string[outputTypes.Length]; for (int i = 0; i < outputTypes.Length; i++) { CodeVariableDeclarationStatement writerDecl = MakeVertexWriterDecl(outputTypes[i], this.GetStaticFactoryName(outputTypes[i])); vertexMethod.Statements.Add(writerDecl); writerNames[i] = writerDecl.Name; } // Add side readers: node.AddSideReaders(vertexMethod); // Generate code based on the node type: switch (node.NodeType) { case QueryNodeType.Where: case QueryNodeType.OrderBy: case QueryNodeType.Distinct: case QueryNodeType.Skip: case QueryNodeType.SkipWhile: case QueryNodeType.Take: case QueryNodeType.TakeWhile: case QueryNodeType.Merge: case QueryNodeType.Select: case QueryNodeType.SelectMany: case QueryNodeType.Zip: case QueryNodeType.GroupBy: case QueryNodeType.BasicAggregate: case QueryNodeType.Aggregate: case QueryNodeType.Contains: case QueryNodeType.Join: case QueryNodeType.GroupJoin: case QueryNodeType.Union: case QueryNodeType.Intersect: case QueryNodeType.Except: case QueryNodeType.RangePartition: case QueryNodeType.HashPartition: case QueryNodeType.Apply: case QueryNodeType.Fork: case QueryNodeType.Dynamic: { Type[] inputTypes = node.InputTypes; string[] sourceNames = new string[inputTypes.Length]; for (int i = 0; i < inputTypes.Length; i++) { CodeVariableDeclarationStatement readerDecl = MakeVertexReaderDecl(inputTypes[i], this.GetStaticFactoryName(inputTypes[i])); vertexMethod.Statements.Add(readerDecl); sourceNames[i] = readerDecl.Name; } string sourceToSink = this.m_vertexCodeGen.AddVertexCode(node, vertexMethod, sourceNames, writerNames); if (sourceToSink != null) { CodeExpression sinkExpr = new CodeMethodInvokeExpression( new CodeVariableReferenceExpression(writerNames[0]), "WriteItemSequence", new CodeVariableReferenceExpression(sourceToSink)); vertexMethod.Statements.Add(sinkExpr); } break; } case QueryNodeType.Super: { string sourceToSink = this.m_vertexCodeGen.AddVertexCode(node, vertexMethod, null, writerNames); if (sourceToSink != null) { CodeExpression sinkExpr = new CodeMethodInvokeExpression( new CodeVariableReferenceExpression(writerNames[0]), "WriteItemSequence", new CodeVariableReferenceExpression(sourceToSink)); vertexMethod.Statements.Add(sinkExpr); } break; } default: { //@@TODO: this should not be reachable. could change to Assert/InvalidOpEx throw new DryadLinqException(DryadLinqErrorCode.Internal, String.Format(SR.AddVertexNotHandled, node.NodeType)); } } string completedMsg = "DryadLinqLog.AddInfo(\"Vertex " + vertexMethod.Name + " completed at {0}\", DateTime.Now.ToString(\"MM/dd/yyyy HH:mm:ss.fff\"))"; vertexMethod.Statements.Add(new CodeSnippetExpression(completedMsg)); // add a catch block CodeCatchClause catchBlock = new CodeCatchClause("e"); CodeTypeReferenceExpression errorReportClass = new CodeTypeReferenceExpression("VertexEnv"); CodeMethodReferenceExpression errorReportMethod = new CodeMethodReferenceExpression(errorReportClass, "ReportVertexError"); CodeVariableReferenceExpression exRef = new CodeVariableReferenceExpression(catchBlock.LocalName); catchBlock.Statements.Add(new CodeMethodInvokeExpression(errorReportMethod, exRef)); tryBlock.CatchClauses.Add(catchBlock); // wrap the entire vertex method in a try/catch block tryBlock.TryStatements.AddRange(vertexMethod.Statements); vertexMethod.Statements.Clear(); vertexMethod.Statements.Add(tryBlock); // Always add "return 0", to make CLR hosting happy... vertexMethod.Statements.Add(new CodeMethodReturnStatement(ZeroExpr)); this.m_dryadVertexClass.Members.Add(vertexMethod); return vertexMethod; }
private void AssignUniqueId(DLinqQueryNode node) { if (node.m_uniqueId == this.m_currentPhaseId) { foreach (Pair<ParameterExpression, DLinqQueryNode> refChild in node.GetReferencedQueries()) { this.AssignUniqueId(refChild.Value); } foreach (DLinqQueryNode child in node.Children) { this.AssignUniqueId(child); } if (node.m_uniqueId == this.m_currentPhaseId) { node.m_uniqueId = this.m_nextVertexId++; // Special treatment for DoWhile DLinqDoWhileNode doWhileNode = node as DLinqDoWhileNode; if (doWhileNode != null) { this.AssignUniqueId(doWhileNode.Body); this.AssignUniqueId(doWhileNode.Cond); this.AssignUniqueId(doWhileNode.BodySource); this.AssignUniqueId(doWhileNode.CondSource1); this.AssignUniqueId(doWhileNode.CondSource2); } // Special treatment for Fork if (node.OutputNode is DLinqForkNode) { foreach (DLinqQueryNode pnode in node.Parents) { if (pnode.m_uniqueId == this.m_currentPhaseId) { pnode.m_uniqueId = this.m_nextVertexId++; } } } } } }
internal static CodeVariableDeclarationStatement MakeVertexParamsDecl(DLinqQueryNode node) { int inputArity = node.InputArity + node.GetReferencedQueries().Count; int outputArity = node.OutputArity; CodeExpression arg1 = new CodePrimitiveExpression(inputArity); CodeExpression arg2 = new CodePrimitiveExpression(outputArity); CodeExpression vertexParamsInitExpr = new CodeObjectCreateExpression("DryadLinqVertexParams", arg1, arg2); CodeVariableDeclarationStatement vertexParamsDecl = new CodeVariableDeclarationStatement("DryadLinqVertexParams", VertexParamName, vertexParamsInitExpr); return vertexParamsDecl; }
private void GetReferencedQueries(DLinqQueryNode curNode, ReferencedQuerySubst subst) { curNode.GetReferencedQueries(subst); DLinqQueryNode[] curChildren = curNode.Children; for (int i = 0; i < curChildren.Length; i++) { DLinqQueryNode child = curChildren[i]; if (this.Contains(child)) { this.GetReferencedQueries(child, subst); } } }