public void PrintGraph(DrGraph graph) { using (StreamWriter sw = new StreamWriter(new FileStream("toplogy.txt", FileMode.Create))) { List <DrStageManager> stages = graph.GetStages(); foreach (DrStageManager s in stages) { List <DrVertex> vertices = s.GetVertexVector(); foreach (DrVertex v in vertices) { sw.Write("{0} <= ", v.GetId()); DrEdgeHolder eh = v.GetInputs(); int n = eh.GetNumberOfEdges(); if (n == 0) { sw.Write("DSC"); } else { for (int i = 0; i < n; i++) { DrVertex vin = eh.GetEdge(i).m_remoteVertex; sw.Write("{0} ", vin.GetId()); } } sw.WriteLine(); } } } }
internal void FinalizeExecution(Query query, DrGraph graph) { SortedDictionary <int, Vertex> queryPlan = query.queryPlan; foreach (KeyValuePair <int, Vertex> kvp in query.queryPlan) { /* used to do CSStream expiration time stuff here */ } }
public DryadLINQApp(DrGraph graph) { m_graph = graph; m_clusterAffinity = true; m_maxAggregateInputs = 150; m_maxAggregateFilterInputs = 32; m_aggregateThreshold = 1024 * 1024 * 1024; // 1GB m_startTime = graph.GetCluster().GetCurrentTimeStamp(); m_identityMapFile = null; m_optionMap = new SortedDictionary <string, OptionDescription>(); AddOptionsToMap(s_dryadLinqOptionArray); }
private DrVertexSet CreateVertexSet(DrGraph graph, DrVertex prototype, int copies) { DrVertexSet result = new DrVertexSet(); for (int i = 0; i < copies; i++) { DrVertex v = prototype.MakeCopy(i); result.Add(v); } return(result); }
private DrVertexSet CreateVertexSet(DrGraph graph, DrInputStreamManager inputStream) { DrVertexSet result = new DrVertexSet(); List <DrStorageVertex> vertices = inputStream.GetVertices(); for (int i = 0; i < vertices.Count; i++) { DrVertex v = vertices[i]; result.Add(v); } return(result); }
private DrVertexSet CreateVertexSet(DrGraph graph, DrOutputStreamManager outputStream, int parts) { DrVertexSet result = new DrVertexSet(); outputStream.SetNumberOfParts(parts); List <DrOutputVertex> vertices = outputStream.GetVertices(); for (int i = 0; i < vertices.Count; i++) { DrVertex v = vertices[i]; result.Add(v); } return(result); }
public DryadLINQApp(DrGraph graph) { m_graph = graph; m_clusterAffinity = true; m_maxAggregateInputs = 150; m_maxAggregateFilterInputs = 32; m_aggregateThreshold = 1024*1024*1024; // 1GB m_startTime = graph.GetCluster().GetCurrentTimeStamp(); m_identityMapFile = null; m_optionMap = new SortedDictionary<string,OptionDescription>(); AddOptionsToMap(s_dryadLinqOptionArray); }
// // Main Dryad LINQ execution stuff // public int ExecLinqToDryad(string[] args) { // // must be at least two arguments (program name and query XML file name) // if (args.Length < 2) { DryadLogger.LogCritical(0, null, "Must provide at least query XML file name."); return(-1); } // // break if --break is included in arguments (and eliminate it, as it is not known downstream) // if (ConsumeSingleArgument("--break", ref args)) { DebugHelper.WaitForDebugger(); } // // parse the XML input, producing a DryadLINQ Query // Query query = new Query(); QueryPlanParser parser = new QueryPlanParser(); if (!parser.ParseQueryXml(args[1], query)) { DryadLogger.LogCritical(0, null, "Invalid query plan"); return(-1); } // // build internal app arguments // List <string> internalArgs = new List <string>(); // // add the XmlExecHost args to the internal app arguments // foreach (string xmlExecHostArg in query.xmlExecHostArgs) { if (xmlExecHostArg == "--break") { DebugHelper.WaitForDebugger(); } else { internalArgs.Add(xmlExecHostArg); } } // // combine internal arguments with any additional arguments received on the command line // don't include argv[0] and argv[1] (program name and query XML file name) // int internalArgc = (int)internalArgs.Count; int externalArgc = args.Length - 2; // don't include argv[0] and argv[1] int combinedArgc = internalArgc + externalArgc; string[] combinedArgv = new string[combinedArgc]; string msg = ""; // internal arguments first for (int i = 0; i < internalArgc; i++) { combinedArgv[i] = internalArgs[i]; msg += String.Format("{0} ", combinedArgv[i]); } // then external arguments for (int i = 0; i < externalArgc; i++) { combinedArgv[i + internalArgc] = args[i + 2]; // don't include argv[0] and argv[1] msg += String.Format("{0} ", combinedArgv[i + internalArgc]); } DryadLogger.LogInformation(null, "Arguments: {0}", msg); string jobClass = "DryadLINQ"; string dryadBinDir = Environment.GetEnvironmentVariable("DRYAD_HOME"); if (String.IsNullOrEmpty(dryadBinDir)) { throw new ApplicationException("DryadLINQ requires the DRYAD_HOME environment variable to be set to the Dryad binary folder."); } string exeName = Path.Combine(dryadBinDir, "VertexHost.exe"); // create app and run it // DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication); DrArtemisLegacyReporter reporter = new DrArtemisLegacyReporter(); p.m_defaultProcessTemplate.GetListenerList().Add(reporter); p.m_defaultVertexTemplate.GetListenerList().Add(reporter); p.m_topologyReporter = reporter; p.m_intermediateCompressionMode = query.intermediateDataCompression; DrGraphExecutor graphExecutor = new DrGraphExecutor(); DrGraph graph = graphExecutor.Initialize(p); if (graph == null) { DryadLogger.LogCritical(0, null, "Failed to initialize Graph Executor"); return(-1); } DryadLINQApp app = new DryadLINQApp(graph); // Initialize with arguments app.SetXmlFileName(args[1]); if (!app.ParseCommandLineFlags(combinedArgv)) { DryadLogger.LogCritical(0, null, "Bad command-line options"); return(-1); } // Build graph from query plan GraphBuilder builder = new GraphBuilder(); builder.BuildGraphFromQuery(app, query); // Run the app DryadLogger.LogInformation(null, "Running the app"); graphExecutor.Run(); DrError exitStatus = graphExecutor.Join(); DryadLogger.LogInformation(null, "Finished running the app"); if (exitStatus == null || exitStatus.m_code == 0) { FinalizeExecution(query, graph); DryadLogger.LogInformation(null, "Application completed successfully."); return(0); } else { DryadLogger.LogCritical(exitStatus.m_code, null, "Application failed with error code 0x{0:X8}.\n", exitStatus.m_code); return(exitStatus.m_code); } }
internal void FinalizeExecution(Query query, DrGraph graph) { SortedDictionary<int, Vertex> queryPlan = query.queryPlan; foreach (KeyValuePair<int, Vertex> kvp in query.queryPlan) { /* used to do CSStream expiration time stuff here */ } }
// // Main Dryad LINQ execution stuff // public int ExecLinqToDryad(Uri dfsDirectory, string[] args, out string errorString) { // // must be at least two arguments (program name and query XML file name) // if (args.Length < 2) { errorString = "Must provide at least query XML file name and VertexHost executable."; DryadLogger.LogCritical(errorString); return(-1); } // // break if --break is included in arguments (and eliminate it, as it is not known downstream) // if (ConsumeSingleArgument("--break", ref args)) { DebugHelper.WaitForDebugger(); } // this is where we ensure the right type of scheduler is registered Microsoft.Research.Dryad.LocalScheduler.Registration.Ensure(); // // parse the XML input, producing a DryadLINQ Query // Query query = new Query(); QueryPlanParser parser = new QueryPlanParser(); if (!parser.ParseQueryXml(args[1], query)) { errorString = "Invalid query plan"; DryadLogger.LogCritical(errorString); return(-1); } // // upload the query plan to the job DFS directory for the benefit of debug tools // if (dfsDirectory != null) { DebugHelper.UploadToDfs(dfsDirectory, args[1]); } // // build internal app arguments // List <string> internalArgs = new List <string>(); // // add the XmlExecHost args to the internal app arguments // foreach (string xmlExecHostArg in query.xmlExecHostArgs) { if (xmlExecHostArg == "--break") { DebugHelper.WaitForDebugger(); } else { internalArgs.Add(xmlExecHostArg); } } // // combine internal arguments with any additional arguments received on the command line // don't include argv[0] and argv[1] (program name and query XML file name) // int internalArgc = (int)internalArgs.Count; int externalArgc = args.Length - 2; // don't include argv[0] and argv[1] int combinedArgc = internalArgc + externalArgc; string[] combinedArgv = new string[combinedArgc]; string msg = ""; // internal arguments first for (int i = 0; i < internalArgc; i++) { combinedArgv[i] = internalArgs[i]; msg += String.Format("{0} ", combinedArgv[i]); } // then external arguments for (int i = 0; i < externalArgc; i++) { combinedArgv[i + internalArgc] = args[i + 2]; // don't include argv[0] and argv[1] msg += String.Format("{0} ", combinedArgv[i + internalArgc]); } DryadLogger.LogInformation(String.Format("Arguments: {0}", msg)); string jobClass = "DryadLINQ"; string exeName = args[0]; // create app and run it // DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication); foreach (DrIReporter reporter in DebugHelper.Reporters()) { p.m_reporters.Add(reporter); } p.m_intermediateCompressionMode = query.intermediateDataCompression; DrGraphExecutor graphExecutor = new DrGraphExecutor(); DrGraph graph = graphExecutor.Initialize(p); if (graph == null) { errorString = "Failed to initialize Graph Executor"; DryadLogger.LogCritical(errorString); return(-1); } DryadLINQApp app = new DryadLINQApp(graph); // Initialize with arguments app.SetXmlFileName(args[1]); if (!app.ParseCommandLineFlags(combinedArgv)) { errorString = "Bad command-line options"; DryadLogger.LogCritical(errorString); return(-1); } // Build graph from query plan GraphBuilder builder = new GraphBuilder(); builder.BuildGraphFromQuery(app, query); // Run the app DryadLogger.LogInformation("Running the app"); graphExecutor.Run(); DrError exitStatus = graphExecutor.Join(); DryadLogger.LogInformation("Finished running the app"); if (exitStatus == null || exitStatus.m_code == 0) { FinalizeExecution(query, graph); DryadLogger.LogInformation("Application completed successfully."); errorString = null; return(0); } else { DryadLogger.LogCritical(String.Format("Application failed with error code 0x{0:X8}.\n", exitStatus.m_code)); errorString = exitStatus.ToFullTextNative(); return(exitStatus.m_code); } }