// // Main Dryad LINQ execution stuff // public int ExecLinqToDryad(string[] args) { // // must be at least two arguments (program name and query XML file name) // if (args.Length < 2) { DryadLogger.LogCritical(0, null, "Must provide at least query XML file name."); return(-1); } // // break if --break is included in arguments (and eliminate it, as it is not known downstream) // if (ConsumeSingleArgument("--break", ref args)) { DebugHelper.WaitForDebugger(); } // // parse the XML input, producing a DryadLINQ Query // Query query = new Query(); QueryPlanParser parser = new QueryPlanParser(); if (!parser.ParseQueryXml(args[1], query)) { DryadLogger.LogCritical(0, null, "Invalid query plan"); return(-1); } // // build internal app arguments // List <string> internalArgs = new List <string>(); // // add the XmlExecHost args to the internal app arguments // foreach (string xmlExecHostArg in query.xmlExecHostArgs) { if (xmlExecHostArg == "--break") { DebugHelper.WaitForDebugger(); } else { internalArgs.Add(xmlExecHostArg); } } // // combine internal arguments with any additional arguments received on the command line // don't include argv[0] and argv[1] (program name and query XML file name) // int internalArgc = (int)internalArgs.Count; int externalArgc = args.Length - 2; // don't include argv[0] and argv[1] int combinedArgc = internalArgc + externalArgc; string[] combinedArgv = new string[combinedArgc]; string msg = ""; // internal arguments first for (int i = 0; i < internalArgc; i++) { combinedArgv[i] = internalArgs[i]; msg += String.Format("{0} ", combinedArgv[i]); } // then external arguments for (int i = 0; i < externalArgc; i++) { combinedArgv[i + internalArgc] = args[i + 2]; // don't include argv[0] and argv[1] msg += String.Format("{0} ", combinedArgv[i + internalArgc]); } DryadLogger.LogInformation(null, "Arguments: {0}", msg); string jobClass = "DryadLINQ"; string dryadBinDir = Environment.GetEnvironmentVariable("DRYAD_HOME"); if (String.IsNullOrEmpty(dryadBinDir)) { throw new ApplicationException("DryadLINQ requires the DRYAD_HOME environment variable to be set to the Dryad binary folder."); } string exeName = Path.Combine(dryadBinDir, "VertexHost.exe"); // create app and run it // DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication); DrArtemisLegacyReporter reporter = new DrArtemisLegacyReporter(); p.m_defaultProcessTemplate.GetListenerList().Add(reporter); p.m_defaultVertexTemplate.GetListenerList().Add(reporter); p.m_topologyReporter = reporter; p.m_intermediateCompressionMode = query.intermediateDataCompression; DrGraphExecutor graphExecutor = new DrGraphExecutor(); DrGraph graph = graphExecutor.Initialize(p); if (graph == null) { DryadLogger.LogCritical(0, null, "Failed to initialize Graph Executor"); return(-1); } DryadLINQApp app = new DryadLINQApp(graph); // Initialize with arguments app.SetXmlFileName(args[1]); if (!app.ParseCommandLineFlags(combinedArgv)) { DryadLogger.LogCritical(0, null, "Bad command-line options"); return(-1); } // Build graph from query plan GraphBuilder builder = new GraphBuilder(); builder.BuildGraphFromQuery(app, query); // Run the app DryadLogger.LogInformation(null, "Running the app"); graphExecutor.Run(); DrError exitStatus = graphExecutor.Join(); DryadLogger.LogInformation(null, "Finished running the app"); if (exitStatus == null || exitStatus.m_code == 0) { FinalizeExecution(query, graph); DryadLogger.LogInformation(null, "Application completed successfully."); return(0); } else { DryadLogger.LogCritical(exitStatus.m_code, null, "Application failed with error code 0x{0:X8}.\n", exitStatus.m_code); return(exitStatus.m_code); } }
// // Main Dryad LINQ execution stuff // public int ExecLinqToDryad(string[] args) { // // must be at least two arguments (program name and query XML file name) // if (args.Length < 2) { DryadLogger.LogCritical(0, null, "Must provide at least query XML file name."); return -1; } // // break if --break is included in arguments (and eliminate it, as it is not known downstream) // if (ConsumeSingleArgument("--break", ref args)) { DebugHelper.WaitForDebugger(); } // // parse the XML input, producing a DryadLINQ Query // Query query = new Query(); QueryPlanParser parser = new QueryPlanParser(); if (!parser.ParseQueryXml(args[1], query)) { DryadLogger.LogCritical(0, null, "Invalid query plan"); return -1; } // // build internal app arguments // List<string> internalArgs = new List<string>(); // // add the XmlExecHost args to the internal app arguments // foreach (string xmlExecHostArg in query.xmlExecHostArgs) { if (xmlExecHostArg == "--break") { DebugHelper.WaitForDebugger(); } else { internalArgs.Add(xmlExecHostArg); } } // // combine internal arguments with any additional arguments received on the command line // don't include argv[0] and argv[1] (program name and query XML file name) // int internalArgc = (int)internalArgs.Count; int externalArgc = args.Length - 2; // don't include argv[0] and argv[1] int combinedArgc = internalArgc + externalArgc; string[] combinedArgv = new string[combinedArgc]; string msg = ""; // internal arguments first for (int i=0; i<internalArgc; i++) { combinedArgv[i] = internalArgs[i]; msg += String.Format("{0} ", combinedArgv[i]); } // then external arguments for (int i = 0; i<externalArgc; i++) { combinedArgv[i+internalArgc] = args[i+2]; // don't include argv[0] and argv[1] msg += String.Format("{0} ", combinedArgv[i+internalArgc]); } DryadLogger.LogInformation(null, "Arguments: {0}", msg); string jobClass = "DryadLINQ"; string dryadBinDir = Environment.GetEnvironmentVariable("DRYAD_HOME"); if (String.IsNullOrEmpty(dryadBinDir)) { throw new ApplicationException("DryadLINQ requires the DRYAD_HOME environment variable to be set to the Dryad binary folder."); } string exeName = Path.Combine(dryadBinDir, "VertexHost.exe"); // create app and run it // DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication); DrArtemisLegacyReporter reporter = new DrArtemisLegacyReporter(); p.m_defaultProcessTemplate.GetListenerList().Add(reporter); p.m_defaultVertexTemplate.GetListenerList().Add(reporter); p.m_topologyReporter = reporter; p.m_intermediateCompressionMode = query.intermediateDataCompression; DrGraphExecutor graphExecutor = new DrGraphExecutor(); DrGraph graph = graphExecutor.Initialize(p); if (graph == null) { DryadLogger.LogCritical(0, null, "Failed to initialize Graph Executor"); return -1; } DryadLINQApp app = new DryadLINQApp(graph); // Initialize with arguments app.SetXmlFileName(args[1]); if (!app.ParseCommandLineFlags(combinedArgv)) { DryadLogger.LogCritical(0, null, "Bad command-line options"); return -1; } // Build graph from query plan GraphBuilder builder = new GraphBuilder(); builder.BuildGraphFromQuery(app, query); // Run the app DryadLogger.LogInformation(null, "Running the app"); graphExecutor.Run(); DrError exitStatus = graphExecutor.Join(); DryadLogger.LogInformation(null, "Finished running the app"); if (exitStatus == null || exitStatus.m_code == 0) { FinalizeExecution(query, graph); DryadLogger.LogInformation(null, "Application completed successfully."); return 0; } else { DryadLogger.LogCritical(exitStatus.m_code, null, "Application failed with error code 0x{0:X8}.\n", exitStatus.m_code); return exitStatus.m_code; } }
// // Main Dryad LINQ execution stuff // public int ExecLinqToDryad(Uri dfsDirectory, string[] args, out string errorString) { // // must be at least two arguments (program name and query XML file name) // if (args.Length < 2) { errorString = "Must provide at least query XML file name and VertexHost executable."; DryadLogger.LogCritical(errorString); return -1; } // // break if --break is included in arguments (and eliminate it, as it is not known downstream) // if (ConsumeSingleArgument("--break", ref args)) { DebugHelper.WaitForDebugger(); } // this is where we ensure the right type of scheduler is registered Microsoft.Research.Dryad.LocalScheduler.Registration.Ensure(); // // parse the XML input, producing a DryadLINQ Query // Query query = new Query(); QueryPlanParser parser = new QueryPlanParser(); if (!parser.ParseQueryXml(args[1], query)) { errorString = "Invalid query plan"; DryadLogger.LogCritical(errorString); return -1; } // // upload the query plan to the job DFS directory for the benefit of debug tools // if (dfsDirectory != null) { DebugHelper.UploadToDfs(dfsDirectory, args[1]); } // // build internal app arguments // List<string> internalArgs = new List<string>(); // // add the XmlExecHost args to the internal app arguments // foreach (string xmlExecHostArg in query.xmlExecHostArgs) { if (xmlExecHostArg == "--break") { DebugHelper.WaitForDebugger(); } else { internalArgs.Add(xmlExecHostArg); } } // // combine internal arguments with any additional arguments received on the command line // don't include argv[0] and argv[1] (program name and query XML file name) // int internalArgc = (int)internalArgs.Count; int externalArgc = args.Length - 2; // don't include argv[0] and argv[1] int combinedArgc = internalArgc + externalArgc; string[] combinedArgv = new string[combinedArgc]; string msg = ""; // internal arguments first for (int i=0; i<internalArgc; i++) { combinedArgv[i] = internalArgs[i]; msg += String.Format("{0} ", combinedArgv[i]); } // then external arguments for (int i = 0; i<externalArgc; i++) { combinedArgv[i+internalArgc] = args[i+2]; // don't include argv[0] and argv[1] msg += String.Format("{0} ", combinedArgv[i+internalArgc]); } DryadLogger.LogInformation(String.Format("Arguments: {0}", msg)); string jobClass = "DryadLINQ"; string exeName = args[0]; // create app and run it // DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication); foreach (DrIReporter reporter in DebugHelper.Reporters()) { p.m_reporters.Add(reporter); } p.m_intermediateCompressionMode = query.intermediateDataCompression; DrGraphExecutor graphExecutor = new DrGraphExecutor(); DrGraph graph = graphExecutor.Initialize(p); if (graph == null) { errorString = "Failed to initialize Graph Executor"; DryadLogger.LogCritical(errorString); return -1; } DryadLINQApp app = new DryadLINQApp(graph); // Initialize with arguments app.SetXmlFileName(args[1]); if (!app.ParseCommandLineFlags(combinedArgv)) { errorString = "Bad command-line options"; DryadLogger.LogCritical(errorString); return -1; } // Build graph from query plan GraphBuilder builder = new GraphBuilder(); builder.BuildGraphFromQuery(app, query); // Run the app DryadLogger.LogInformation("Running the app"); graphExecutor.Run(); DrError exitStatus = graphExecutor.Join(); DryadLogger.LogInformation("Finished running the app"); if (exitStatus == null || exitStatus.m_code == 0) { FinalizeExecution(query, graph); DryadLogger.LogInformation("Application completed successfully."); errorString = null; return 0; } else { DryadLogger.LogCritical(String.Format("Application failed with error code 0x{0:X8}.\n", exitStatus.m_code)); errorString = exitStatus.ToFullTextNative(); return exitStatus.m_code; } }
// // Main Dryad LINQ execution stuff // public int ExecLinqToDryad(Uri dfsDirectory, string[] args, out string errorString) { // // must be at least two arguments (program name and query XML file name) // if (args.Length < 2) { errorString = "Must provide at least query XML file name and VertexHost executable."; DryadLogger.LogCritical(errorString); return(-1); } // // break if --break is included in arguments (and eliminate it, as it is not known downstream) // if (ConsumeSingleArgument("--break", ref args)) { DebugHelper.WaitForDebugger(); } // this is where we ensure the right type of scheduler is registered Microsoft.Research.Dryad.LocalScheduler.Registration.Ensure(); // // parse the XML input, producing a DryadLINQ Query // Query query = new Query(); QueryPlanParser parser = new QueryPlanParser(); if (!parser.ParseQueryXml(args[1], query)) { errorString = "Invalid query plan"; DryadLogger.LogCritical(errorString); return(-1); } // // upload the query plan to the job DFS directory for the benefit of debug tools // if (dfsDirectory != null) { DebugHelper.UploadToDfs(dfsDirectory, args[1]); } // // build internal app arguments // List <string> internalArgs = new List <string>(); // // add the XmlExecHost args to the internal app arguments // foreach (string xmlExecHostArg in query.xmlExecHostArgs) { if (xmlExecHostArg == "--break") { DebugHelper.WaitForDebugger(); } else { internalArgs.Add(xmlExecHostArg); } } // // combine internal arguments with any additional arguments received on the command line // don't include argv[0] and argv[1] (program name and query XML file name) // int internalArgc = (int)internalArgs.Count; int externalArgc = args.Length - 2; // don't include argv[0] and argv[1] int combinedArgc = internalArgc + externalArgc; string[] combinedArgv = new string[combinedArgc]; string msg = ""; // internal arguments first for (int i = 0; i < internalArgc; i++) { combinedArgv[i] = internalArgs[i]; msg += String.Format("{0} ", combinedArgv[i]); } // then external arguments for (int i = 0; i < externalArgc; i++) { combinedArgv[i + internalArgc] = args[i + 2]; // don't include argv[0] and argv[1] msg += String.Format("{0} ", combinedArgv[i + internalArgc]); } DryadLogger.LogInformation(String.Format("Arguments: {0}", msg)); string jobClass = "DryadLINQ"; string exeName = args[0]; // create app and run it // DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication); foreach (DrIReporter reporter in DebugHelper.Reporters()) { p.m_reporters.Add(reporter); } p.m_intermediateCompressionMode = query.intermediateDataCompression; DrGraphExecutor graphExecutor = new DrGraphExecutor(); DrGraph graph = graphExecutor.Initialize(p); if (graph == null) { errorString = "Failed to initialize Graph Executor"; DryadLogger.LogCritical(errorString); return(-1); } DryadLINQApp app = new DryadLINQApp(graph); // Initialize with arguments app.SetXmlFileName(args[1]); if (!app.ParseCommandLineFlags(combinedArgv)) { errorString = "Bad command-line options"; DryadLogger.LogCritical(errorString); return(-1); } // Build graph from query plan GraphBuilder builder = new GraphBuilder(); builder.BuildGraphFromQuery(app, query); // Run the app DryadLogger.LogInformation("Running the app"); graphExecutor.Run(); DrError exitStatus = graphExecutor.Join(); DryadLogger.LogInformation("Finished running the app"); if (exitStatus == null || exitStatus.m_code == 0) { FinalizeExecution(query, graph); DryadLogger.LogInformation("Application completed successfully."); errorString = null; return(0); } else { DryadLogger.LogCritical(String.Format("Application failed with error code 0x{0:X8}.\n", exitStatus.m_code)); errorString = exitStatus.ToFullTextNative(); return(exitStatus.m_code); } }