// // Main Dryad LINQ execution stuff // public int ExecLinqToDryad(Uri dfsDirectory, string[] args, out string errorString) { // // must be at least two arguments (program name and query XML file name) // if (args.Length < 2) { errorString = "Must provide at least query XML file name and VertexHost executable."; DryadLogger.LogCritical(errorString); return -1; } // // break if --break is included in arguments (and eliminate it, as it is not known downstream) // if (ConsumeSingleArgument("--break", ref args)) { DebugHelper.WaitForDebugger(); } // this is where we ensure the right type of scheduler is registered Microsoft.Research.Dryad.LocalScheduler.Registration.Ensure(); // // parse the XML input, producing a DryadLINQ Query // Query query = new Query(); QueryPlanParser parser = new QueryPlanParser(); if (!parser.ParseQueryXml(args[1], query)) { errorString = "Invalid query plan"; DryadLogger.LogCritical(errorString); return -1; } // // upload the query plan to the job DFS directory for the benefit of debug tools // if (dfsDirectory != null) { DebugHelper.UploadToDfs(dfsDirectory, args[1]); } // // build internal app arguments // List<string> internalArgs = new List<string>(); // // add the XmlExecHost args to the internal app arguments // foreach (string xmlExecHostArg in query.xmlExecHostArgs) { if (xmlExecHostArg == "--break") { DebugHelper.WaitForDebugger(); } else { internalArgs.Add(xmlExecHostArg); } } // // combine internal arguments with any additional arguments received on the command line // don't include argv[0] and argv[1] (program name and query XML file name) // int internalArgc = (int)internalArgs.Count; int externalArgc = args.Length - 2; // don't include argv[0] and argv[1] int combinedArgc = internalArgc + externalArgc; string[] combinedArgv = new string[combinedArgc]; string msg = ""; // internal arguments first for (int i=0; i<internalArgc; i++) { combinedArgv[i] = internalArgs[i]; msg += String.Format("{0} ", combinedArgv[i]); } // then external arguments for (int i = 0; i<externalArgc; i++) { combinedArgv[i+internalArgc] = args[i+2]; // don't include argv[0] and argv[1] msg += String.Format("{0} ", combinedArgv[i+internalArgc]); } DryadLogger.LogInformation(String.Format("Arguments: {0}", msg)); string jobClass = "DryadLINQ"; string exeName = args[0]; // create app and run it // DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication); foreach (DrIReporter reporter in DebugHelper.Reporters()) { p.m_reporters.Add(reporter); } p.m_intermediateCompressionMode = query.intermediateDataCompression; DrGraphExecutor graphExecutor = new DrGraphExecutor(); DrGraph graph = graphExecutor.Initialize(p); if (graph == null) { errorString = "Failed to initialize Graph Executor"; DryadLogger.LogCritical(errorString); return -1; } DryadLINQApp app = new DryadLINQApp(graph); // Initialize with arguments app.SetXmlFileName(args[1]); if (!app.ParseCommandLineFlags(combinedArgv)) { errorString = "Bad command-line options"; DryadLogger.LogCritical(errorString); return -1; } // Build graph from query plan GraphBuilder builder = new GraphBuilder(); builder.BuildGraphFromQuery(app, query); // Run the app DryadLogger.LogInformation("Running the app"); graphExecutor.Run(); DrError exitStatus = graphExecutor.Join(); DryadLogger.LogInformation("Finished running the app"); if (exitStatus == null || exitStatus.m_code == 0) { FinalizeExecution(query, graph); DryadLogger.LogInformation("Application completed successfully."); errorString = null; return 0; } else { DryadLogger.LogCritical(String.Format("Application failed with error code 0x{0:X8}.\n", exitStatus.m_code)); errorString = exitStatus.ToFullTextNative(); return exitStatus.m_code; } }
public bool ParseQueryXml(string queryPlanFileName, Query query) { XmlNode version = null; XmlDocument queryPlanDoc = new XmlDocument(); // // Load query plan document // try { queryPlanDoc.Load(queryPlanFileName); } catch (Exception e) { DryadLogger.LogCritical(String.Format("Failed to load query plan: {0}: {1}", queryPlanFileName, e.ToString())); return false; } // // Get DryadLinqVersion - absence used to indicate Argentia query plan // try { version = queryPlanDoc.DocumentElement.SelectSingleNode("DryadLinqVersion"); } catch (System.Xml.XPath.XPathException e) { DryadLogger.LogCritical(String.Format("Failed to select node DryadLinqVersion from query plan: {0}: {1}", queryPlanFileName, e.ToString())); return false; } if (version == null) { DryadLogger.LogCritical(String.Format("Missing element 'DryadLinqVersion' in query plan: {0}", queryPlanFileName)); return false; } // // Parse query plan XML doc into Query // try { ParseQueryXmlLinqToDryad(queryPlanDoc, query); } catch (Exception e) { DryadLogger.LogCritical(String.Format("Failed to parse query plan: {0}: {1}", queryPlanFileName, e.ToString())); return false; } #if REMOVE_ARGENTIA else // If (version == null), Argentia query plan { // Add the namespace. XmlNamespaceManager nsmgr = new XmlNamespaceManager(queryPlanDoc.NameTable); nsmgr.AddNamespace("qp", "http://microsoft.com/PCP/Argentia/QueryPlan.xsd"); version = queryPlanDoc.DocumentElement.SelectSingleNode("qp:RuntimeVersion", nsmgr); if (version != null) { ParseQueryXmlArgentia(queryPlanDoc, query); } else { DryadLogger.LogCritical(0, null, "Unknown query plan format."); return false; } } #endif return true; }
internal void FinalizeExecution(Query query, DrGraph graph) { SortedDictionary<int, Vertex> queryPlan = query.queryPlan; foreach (KeyValuePair<int, Vertex> kvp in query.queryPlan) { /* used to do CSStream expiration time stuff here */ } }
private void ParseQueryXmlLinqToDryad(XmlDocument queryPlanDoc, Query query) { XmlElement root = queryPlanDoc.DocumentElement; // // Query globals // query.queryPlan = new SortedDictionary<int, Vertex>(); query.compilerversion = root.SelectSingleNode("DryadLinqVersion").InnerText; query.clusterName = root.SelectSingleNode("ClusterName").InnerText; query.visualization = root.SelectSingleNode("Visualization").InnerText; // Compression scheme for intermediate data XmlNode compressionNode = root.SelectSingleNode("IntermediateDataCompression"); if (compressionNode != null) { query.intermediateDataCompression = Convert.ToInt32(compressionNode.InnerText); } // // XmlExecHost arguments // XmlNodeList nodes = root.SelectSingleNode("XmlExecHostArgs").ChildNodes; query.xmlExecHostArgs = new string[nodes.Count]; for (int index=0; index<nodes.Count; index++) { query.xmlExecHostArgs[index] = nodes[index].InnerText; } // // Get Speculative duplication flag - default is enabled (true) // XmlNode duplicationNode = root.SelectSingleNode("EnableSpeculativeDuplication"); if (duplicationNode != null) { bool dupFlag; if (bool.TryParse(duplicationNode.InnerText, out dupFlag)) { query.enableSpeculativeDuplication = dupFlag; } } nodes = root.SelectSingleNode("QueryPlan").ChildNodes; // // Need to remember the conection operator for use when the // predecessors are being created. // string[] vertexConnectionOperator = new string[nodes.Count]; for (int index=0; index<nodes.Count; index++) { vertexConnectionOperator[index] = ""; } for (int index=0; index<nodes.Count; index++) { Vertex vertex = new Vertex(); // // Vertex globals // string uniqueId = nodes[index].SelectSingleNode("UniqueId").InnerText; string vertexType = nodes[index].SelectSingleNode("Type").InnerText; string name = nodes[index].SelectSingleNode("Name").InnerText; string partitions = nodes[index].SelectSingleNode("Partitions").InnerText; string channelType = nodes[index].SelectSingleNode("ChannelType").InnerText; // // Need to remember the conection operator for use when the // predecessors are being created. // vertexConnectionOperator[index] = nodes[index].SelectSingleNode("ConnectionOperator").InnerText; vertex.uniqueId = Convert.ToInt32(uniqueId); vertex.name = name; vertex.type = GetVertexType(vertexType); vertex.partitions = Convert.ToInt32(partitions); XmlNode dynamicManager = nodes[index].SelectSingleNode("DynamicManager"); string dmType = dynamicManager.SelectSingleNode("Type").InnerText; vertex.dynamicManager = new DynamicManager(); vertex.dynamicManager.type = GetDynamicManagerType(dmType); if (vertex.dynamicManager.type == DynamicManager.Type.FULLAGGR) { string levels = dynamicManager.SelectSingleNode("AggregationLevels").InnerText; vertex.dynamicManager.aggregationLevels = Convert.ToInt32(levels); } if (vertex.dynamicManager.type == DynamicManager.Type.RANGEDISTRIBUTOR) { string sampleRate = dynamicManager.SelectSingleNode("SampleRate").InnerText; string vertexId = dynamicManager.SelectSingleNode("VertexId").InnerText; vertex.dynamicManager.sampleRate = Convert.ToDouble(sampleRate); vertex.dynamicManager.splitVertexId = Convert.ToInt32(vertexId); } else { XmlNodeList entries = dynamicManager.SelectNodes("Entry"); vertex.dynamicManager.assemblyNames = new string[entries.Count]; vertex.dynamicManager.classNames = new string[entries.Count]; vertex.dynamicManager.methodNames = new string[entries.Count]; for (int entryIndex = 0; entryIndex < entries.Count; entryIndex++) { vertex.dynamicManager.assemblyNames[entryIndex] = entries[entryIndex].SelectSingleNode("AssemblyName").InnerText; vertex.dynamicManager.classNames[entryIndex] = entries[entryIndex].SelectSingleNode("ClassName").InnerText; vertex.dynamicManager.methodNames[entryIndex] = entries[entryIndex].SelectSingleNode("MethodName").InnerText; } } if (vertex.type == Vertex.Type.INPUTTABLE) { XmlNode storageSet = nodes[index].SelectSingleNode("StorageSet"); string ioType = storageSet.SelectSingleNode("Type").InnerText; vertex.info = new VertexInfo(); vertex.info.ioType = GetIoType(ioType); XmlNodeList storageUris = storageSet.SelectNodes("SourceURI"); vertex.info.sources = new string[storageUris.Count]; for (int indexStorageUri=0; indexStorageUri<storageUris.Count; indexStorageUri++) { vertex.info.sources[indexStorageUri] = storageUris[indexStorageUri].InnerText; } } else if ( vertex.type == Vertex.Type.OUTPUTTABLE ) { XmlNode storageSet = nodes[index].SelectSingleNode("StorageSet"); string ioType = storageSet.SelectSingleNode("Type").InnerText; vertex.info = new VertexInfo(); vertex.info.ioType = GetIoType(ioType); string source = storageSet.SelectSingleNode("SinkURI").InnerText; vertex.info.sources = new string[1] { source }; if (vertex.info.ioType == VertexInfo.IOType.PARTITIONEDFILE ) { vertex.info.partitionUncPath = storageSet.SelectSingleNode("PartitionUncPath").InnerText; } XmlNode temporary = storageSet.SelectSingleNode("IsTemporary"); if (temporary != null) { if (bool.TryParse(temporary.InnerXml, out vertex.info.isTemporary) == false) { throw new LinqToDryadException(String.Format("Invalid value for IsTemporary: {0}", temporary.InnerXml)); } } else { vertex.info.isTemporary = false; } XmlNode recordType = storageSet.SelectSingleNode("RecordType"); if (recordType != null) { vertex.info.recordType = recordType.InnerXml; } XmlNode outputCompressionScheme = storageSet.SelectSingleNode("OutputCompressionScheme"); if (outputCompressionScheme != null) { if (int.TryParse(outputCompressionScheme.InnerXml, out vertex.info.compressionScheme) == false) { throw new LinqToDryadException(String.Format("Invalid value for OutputCompressionScheme: {0}", outputCompressionScheme.InnerXml)); } } else { vertex.info.compressionScheme = 0; // TODO: Change to Enum } } else /* JOIN etc. */ { XmlNode entry = nodes[index].SelectSingleNode("Entry"); vertex.info = new VertexInfo(); vertex.info.assemblyName = entry.SelectSingleNode("AssemblyName").InnerText; vertex.info.className = entry.SelectSingleNode("ClassName").InnerText; vertex.info.methodName = entry.SelectSingleNode("MethodName").InnerText; } // // everybody except inputs have children // if (vertex.type != Vertex.Type.INPUTTABLE) { XmlNodeList children = nodes[index].SelectSingleNode("Children").ChildNodes; vertex.info.predecessors = new Predecessor[children.Count]; for (int indexChild = 0; indexChild < children.Count; indexChild++) { int childId = Convert.ToInt32(children[indexChild].SelectSingleNode("UniqueId").InnerText); string childConstraint = children[indexChild].SelectSingleNode("AffinityConstraint").InnerText; vertex.info.predecessors[indexChild] = new Predecessor(); vertex.info.predecessors[indexChild].uniqueId = childId; vertex.info.predecessors[indexChild].connectionOperator = GetConnectionOperator(vertexConnectionOperator[childId]); vertex.info.predecessors[indexChild].channelType = GetChannelType(channelType); vertex.info.predecessors[indexChild].constraint = GetAffinityConstraint(childConstraint); } } // // In this parser the only way to have an optional Tag is immediately before consuming // a close of an outer tag. So we have to look for machines tag here. // if ((vertex.type != Vertex.Type.INPUTTABLE) && (vertex.type != Vertex.Type.OUTPUTTABLE)) { XmlNode machinesRoot = nodes[index].SelectSingleNode("Machines"); if (machinesRoot != null) { XmlNodeList machines = machinesRoot.ChildNodes; vertex.machines = new string[machines.Count]; for (int indexMachine = 0; indexMachine < machines.Count; indexMachine++) { vertex.machines[indexMachine] = machines[indexMachine].SelectSingleNode("Machine").InnerText; } } } // // Add the vertex // query.queryPlan.Add(vertex.uniqueId, vertex); } }