static void ExceptionHandler(object sender, UnhandledExceptionEventArgs args) { int result = unchecked ((int)LinqToDryadException.E_FAIL); Exception e = args.ExceptionObject as Exception; if (e != null) { result = System.Runtime.InteropServices.Marshal.GetHRForException(e); DryadLogger.LogCritical(0, e); } DebugHelper.StopLogging(result); if (Debugger.IsAttached) { Debugger.Break(); } else { DrLogging.WriteMiniDump(); } // We need to Exit, since other threads in the GM // are likely to still be running. Environment.Exit(result); }
public static void InitializeLogging(DateTime startTime) { if (!loggingInitialized) { lock (syncRoot) { if (!loggingInitialized) { // Initialize text-based tracing string traceFile = Path.Combine(Directory.GetCurrentDirectory(), "GraphManagerTrace.txt"); DryadLogger.Start(traceFile); // Initialize Graph Manager's internal logging DrLogging.Initialize(); DebugHelper.SetLogType(); // Report start time to Artemis - must come after // DrLogging is initialized so stdout is redirected DrArtemisLegacyReporter.ReportStart((ulong)startTime.Ticks); loggingInitialized = true; } } } }
public bool ParseQueryXml(string queryPlanFileName, Query query) { XmlNode version = null; XmlDocument queryPlanDoc = new XmlDocument(); // // Load query plan document // try { queryPlanDoc.Load(queryPlanFileName); } catch (Exception e) { DryadLogger.LogCritical(0, e, "Failed to load query plan: {0}", queryPlanFileName); return(false); } // // Get DryadLinqVersion - absence used to indicate Argentia query plan // try { version = queryPlanDoc.DocumentElement.SelectSingleNode("DryadLinqVersion"); } catch (System.Xml.XPath.XPathException e) { DryadLogger.LogCritical(0, e, "Failed to select node DryadLinqVersion from query plan: {0}", queryPlanFileName); return(false); } if (version == null) { DryadLogger.LogCritical(0, null, "Missing element 'DryadLinqVersion' in query plan: {0}", queryPlanFileName); return(false); } // // Parse query plan XML doc into Query // try { ParseQueryXmlLinqToDryad(queryPlanDoc, query); } catch (Exception e) { DryadLogger.LogCritical(0, e, "Failed to parse query plan: {0}", queryPlanFileName); return(false); }
public static void StopLogging(int retCode) { if (loggingInitialized) { lock (syncRoot) { if (loggingInitialized) { // Report stop time to Artemis DrArtemisLegacyReporter.ReportStop(unchecked ((uint)retCode)); // Shutdown Graph Manager's internal logging DrLogging.ShutDown(unchecked ((uint)retCode)); // Shutdown text-based tracing DryadLogger.Stop(); loggingInitialized = false; } } } }
static int Main(string[] args) { Console.WriteLine("{0} starting up in {1}", Environment.CommandLine, Environment.CurrentDirectory); bool waitForDebugger = false; List <string> tempArgs = new List <string>(); // Record start time so we can report it once logging has been initialized DateTime startTime = DateTime.Now.ToLocalTime(); // Set unhandled exception handler to catch anything thrown from // Microsoft.Hpc.Query.GraphManager.dll AppDomain currentDomain = AppDomain.CurrentDomain; currentDomain.UnhandledException += new UnhandledExceptionEventHandler(ExceptionHandler); // // Add executable name to beginning of args // to make graph manager libraries happy // tempArgs.Add("YarnQueryGraphManager"); foreach (string arg in args) { if (String.Compare(arg, "--break", StringComparison.OrdinalIgnoreCase) == 0) { waitForDebugger = true; } else { tempArgs.Add(arg); } } args = tempArgs.ToArray(); if (waitForDebugger) { DebugHelper.WaitForDebugger(); } /* Yarn removes the need for this * * // * // Create job directory and copy resources * // * string resources = Environment.GetEnvironmentVariable("XC_RESOURCEFILES"); * if (ExecutionHelper.InitializeForJobExecution(resources) == false) * { * return 1; * } * * // * // Set current directory to working directory * // * Directory.SetCurrentDirectory(ProcessPathHelper.JobPath); */ // // Configure tracing // DebugHelper.InitializeLogging(startTime); // Ensure that there is a jvm.dll on the path. string pathString = Environment.GetEnvironmentVariable("PATH"); var pathDirs = pathString.Split(';'); bool found = false; foreach (var dir in pathDirs) { string targetFile = Path.Combine(dir, "jvm.dll"); if (File.Exists(targetFile)) { found = true; break; } } if (!found) { string javaHome = Environment.GetEnvironmentVariable("JAVA_HOME"); if (String.IsNullOrEmpty(javaHome)) { throw new ApplicationException("DryadLINQ requires the JAVA_HOME environment variable to be set or the jvm.dll to be on the path."); } var jvmPath = ";" + Path.Combine(javaHome, "jre", "bin", "server"); pathString += jvmPath; Environment.SetEnvironmentVariable("PATH", pathString); } // // Run the Graph Manager // int retCode = 0; try { LinqToDryadJM jm = new LinqToDryadJM(); retCode = jm.Run(args); } catch (Exception e) { retCode = System.Runtime.InteropServices.Marshal.GetHRForException(e); if (retCode == 0) { DryadLogger.LogCritical(0, e); retCode = unchecked ((int)LinqToDryadException.E_FAIL); } else { DryadLogger.LogCritical(retCode, e); } System.Threading.Thread.Sleep(10 * 60 * 1000); } DebugHelper.StopLogging(retCode); // NOTE: We don't want to log critical errors twice, so here we're assuming that // if the GM exited "gracefully" and returned an error code instead of throwing // an exception, that it has already logged the error. if (retCode != 0) { // If the Graph Manager already started executing, we need to exit the process. // Exiting the thread (returning from Main) will not necessarily cause the GM's // worker threads to exit // TODO: Consider deleting temp output stream from DSC // requires that we have access to the URI at this point, though Environment.Exit(retCode); } // // Cleanup all vertex tasks in case any became // unreachable. // return(retCode); }
// // Main Dryad LINQ execution stuff // public int ExecLinqToDryad(string[] args) { // // must be at least two arguments (program name and query XML file name) // if (args.Length < 2) { DryadLogger.LogCritical(0, null, "Must provide at least query XML file name."); return(-1); } // // break if --break is included in arguments (and eliminate it, as it is not known downstream) // if (ConsumeSingleArgument("--break", ref args)) { DebugHelper.WaitForDebugger(); } // // parse the XML input, producing a DryadLINQ Query // Query query = new Query(); QueryPlanParser parser = new QueryPlanParser(); if (!parser.ParseQueryXml(args[1], query)) { DryadLogger.LogCritical(0, null, "Invalid query plan"); return(-1); } // // build internal app arguments // List <string> internalArgs = new List <string>(); // // add the XmlExecHost args to the internal app arguments // foreach (string xmlExecHostArg in query.xmlExecHostArgs) { if (xmlExecHostArg == "--break") { DebugHelper.WaitForDebugger(); } else { internalArgs.Add(xmlExecHostArg); } } // // combine internal arguments with any additional arguments received on the command line // don't include argv[0] and argv[1] (program name and query XML file name) // int internalArgc = (int)internalArgs.Count; int externalArgc = args.Length - 2; // don't include argv[0] and argv[1] int combinedArgc = internalArgc + externalArgc; string[] combinedArgv = new string[combinedArgc]; string msg = ""; // internal arguments first for (int i = 0; i < internalArgc; i++) { combinedArgv[i] = internalArgs[i]; msg += String.Format("{0} ", combinedArgv[i]); } // then external arguments for (int i = 0; i < externalArgc; i++) { combinedArgv[i + internalArgc] = args[i + 2]; // don't include argv[0] and argv[1] msg += String.Format("{0} ", combinedArgv[i + internalArgc]); } DryadLogger.LogInformation(null, "Arguments: {0}", msg); string jobClass = "DryadLINQ"; string dryadBinDir = Environment.GetEnvironmentVariable("DRYAD_HOME"); if (String.IsNullOrEmpty(dryadBinDir)) { throw new ApplicationException("DryadLINQ requires the DRYAD_HOME environment variable to be set to the Dryad binary folder."); } string exeName = Path.Combine(dryadBinDir, "VertexHost.exe"); // create app and run it // DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication); DrArtemisLegacyReporter reporter = new DrArtemisLegacyReporter(); p.m_defaultProcessTemplate.GetListenerList().Add(reporter); p.m_defaultVertexTemplate.GetListenerList().Add(reporter); p.m_topologyReporter = reporter; p.m_intermediateCompressionMode = query.intermediateDataCompression; DrGraphExecutor graphExecutor = new DrGraphExecutor(); DrGraph graph = graphExecutor.Initialize(p); if (graph == null) { DryadLogger.LogCritical(0, null, "Failed to initialize Graph Executor"); return(-1); } DryadLINQApp app = new DryadLINQApp(graph); // Initialize with arguments app.SetXmlFileName(args[1]); if (!app.ParseCommandLineFlags(combinedArgv)) { DryadLogger.LogCritical(0, null, "Bad command-line options"); return(-1); } // Build graph from query plan GraphBuilder builder = new GraphBuilder(); builder.BuildGraphFromQuery(app, query); // Run the app DryadLogger.LogInformation(null, "Running the app"); graphExecutor.Run(); DrError exitStatus = graphExecutor.Join(); DryadLogger.LogInformation(null, "Finished running the app"); if (exitStatus == null || exitStatus.m_code == 0) { FinalizeExecution(query, graph); DryadLogger.LogInformation(null, "Application completed successfully."); return(0); } else { DryadLogger.LogCritical(exitStatus.m_code, null, "Application failed with error code 0x{0:X8}.\n", exitStatus.m_code); return(exitStatus.m_code); } }
public bool ParseCommandLineFlags(string[] args) { bool retVal = true; for (int index = 0; index < args.Length; index++) { string arg = args[index].Substring(args[index].IndexOf("-") + 1); OptionDescription option = null; m_optionMap.TryGetValue(arg, out option); if (option == null) { retVal = false; break; } int optionIndex = option.m_optionIndex; switch (optionIndex) { case (int)DryadLINQAppOptions.BDJAO_AMD64: case (int)DryadLINQAppOptions.BDJAO_I386: case (int)DryadLINQAppOptions.BDJAO_Retail: case (int)DryadLINQAppOptions.BDJAO_Debug: break; case (int)DryadLINQAppOptions.DNAO_MaxAggregateInputs: if ((index + 1) >= args.Length) { DryadLogger.LogCritical(0, null, "The argument for option '{0}' was missing.\n", args[index]); retVal = false; } else { int maxInputs; if (!Int32.TryParse(args[index + 1], out maxInputs)) { DryadLogger.LogCritical(0, null, "The argument '{0}' for option '{1}' could not be parsed as an integer.\n", args[index + 1], args[index]); retVal = false; } else { m_maxAggregateInputs = maxInputs; index++; } } break; case (int)DryadLINQAppOptions.DNAO_MaxAggregateFilterInputs: if ((index + 1) >= args.Length) { DryadLogger.LogCritical(0, null, "The argument for option '{0}' was missing.\n", args[index]); retVal = false; } else { int maxInputs; if (!Int32.TryParse(args[index + 1], out maxInputs)) { DryadLogger.LogCritical(0, null, "The argument '{0}' for option '{1}' could not be parsed as an integer.\n", args[index + 1], args[index]); retVal = false; } else { m_maxAggregateFilterInputs = maxInputs; index++; } } break; case (int)DryadLINQAppOptions.DNAO_AggregateThreshold: if ((index + 1) >= args.Length) { DryadLogger.LogCritical(0, null, "The argument for option '{0}' was missing.\n", args[index]); retVal = false; } else { UInt64 threshold; if (!UInt64.TryParse(args[index + 1], out threshold)) { DryadLogger.LogCritical(0, null, "The argument '{0}' for option '{1}' could not be parsed as a UIN64.\n", args[index + 1], args[index]); retVal = false; } else { m_aggregateThreshold = threshold; index++; } } break; case (int)DryadLINQAppOptions.DNAO_NoClusterAffinity: m_clusterAffinity = false; break; default: DryadLogger.LogCritical(0, null, "Unknown command-line option {0}\n", optionIndex); retVal = false; break; } } if (!retVal) { PrintUsage(args[0]); } return(retVal); }
private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName) { DrInputStreamManager s; int err = 0; DryadLogger.LogMethodEntry(inputName); if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE) { DrPartitionInputStream input = new DrPartitionInputStream(); err = input.Open(app.GetUniverse(), info.sources[0]); if (!SUCCEEDED(err)) { string msg = String.Format("Could not read DSC input file {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); s = inputManager; } //else if ( info.ioType == VertexInfo.IOType.STREAM ) //{ // DrDscInputStream input = new DrDscInputStream(); // DryadLogger.LogInformation("Create input node", "Opening DSC input fileset"); // err = input.Open(app.GetUniverse(), info.sources[0]); // if (!SUCCEEDED(err)) // { // string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]); // throw new LinqToDryadException(msg, err); // } // DryadLogger.LogInformation("Create input node", "Opened DSC input fileset"); // DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); // DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); // s = inputManager; //} else if (info.ioType == VertexInfo.IOType.HDFS_STREAM) { DrHdfsInputStream input = new DrHdfsInputStream(); DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset"); err = input.Open(app.GetUniverse(), info.sources[0]); if (!SUCCEEDED(err)) { string msg = String.Format("Could not read HDFS input fileset {0}", info.sources[0]); throw new LinqToDryadException(msg, err); } DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset"); DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName); DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage); s = inputManager; } else { string msg = String.Format("Unknown input type {0}", info.ioType); throw new LinqToDryadException(msg); } DryadLogger.LogMethodExit(); return(s); }
public void BuildGraphFromQuery(DryadLINQApp app, Query query) { // set configurable properties int highThreshold = app.GetMaxAggregateInputs(); int lowThreshold = 16; UInt64 highDataThreshold = (UInt64)app.GetAggregateThreshold(); UInt64 lowDataThreshold = (3 * highDataThreshold) / 4; UInt64 maxSingleDataThreshold = highDataThreshold / 2; int aggFilterThreshold = app.GetMaxAggregateFilterInputs(); // use a graph stage map to store the vertices as they are created, grouped by stage. Dictionary <int, GraphStageInfo> graphStageMap = new Dictionary <int, GraphStageInfo>(); DryadLogger.LogInformation("Build Graph From Query", "Building graph"); // // Create a set of vertices for each vertex (stage) in the query plan // DryadLogger.LogInformation("Build Graph From Query", "Adding vertices"); foreach (KeyValuePair <int, Vertex> kvp in query.queryPlan) { Vertex v = kvp.Value; GraphStageInfo value = null; if (!graphStageMap.TryGetValue(v.uniqueId, out value)) { DryadLogger.LogInformation("Build Graph From Query", "Adding vertices for stage {0}", v.name); CreateVertexSet(v, app, query, graphStageMap); } } // // Add dynamic stage managers // DryadLogger.LogInformation("Build Graph From Query", "Adding stage managers"); foreach (KeyValuePair <int, GraphStageInfo> kvp in graphStageMap) { Vertex v = kvp.Value.vertex; // //There are no dynamic managers // if (v.dynamicManager == null) { continue; } DrStageManager newManager = kvp.Value.stageManager; // newManager DrGraphParameters parameters = app.GetGraph().GetParameters(); string stdVertexName = "MW"; string cpyVertexName = "CP"; if (v.type != Vertex.Type.INPUTTABLE && v.type != Vertex.Type.CONCAT) { if (v.dynamicManager.type == DynamicManager.Type.SPLITTER) { if (v.info.predecessors.Length == 1) { DrPipelineSplitManager splitter = new DrPipelineSplitManager(); newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, splitter); } else { DrSemiPipelineSplitManager splitter = new DrSemiPipelineSplitManager(); newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, splitter); } } else if (v.dynamicManager.type == DynamicManager.Type.PARTIALAGGR) { DrDynamicAggregateManager dynamicMerge = new DrDynamicAggregateManager(); dynamicMerge.SetGroupingSettings(0, 0); dynamicMerge.SetMachineGroupingSettings(2, aggFilterThreshold); dynamicMerge.SetDataGroupingSettings(lowDataThreshold, highDataThreshold, maxSingleDataThreshold); dynamicMerge.SetSplitAfterGrouping(true); foreach (Predecessor p in v.info.predecessors) { newManager.AddDynamicConnectionManager(graphStageMap[p.uniqueId].stageManager, dynamicMerge); } } else if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR || v.dynamicManager.type == DynamicManager.Type.HASHDISTRIBUTOR) { int idx = 0; int sz = v.dynamicManager.assemblyNames == null ? 0 : v.dynamicManager.assemblyNames.Length; DrDynamicAggregateManager dynamicMerge = new DrDynamicAggregateManager(); if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR || sz > 1) { dynamicMerge = new DrDynamicAggregateManager(); string name = v.dynamicManager.methodNames[idx]; DrManagerBase newStage = new DrManagerBase(app.GetGraph(), name); DrActiveVertex mergeVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate); mergeVertex.AddArgument(stdVertexName); mergeVertex.AddArgument(v.dynamicManager.assemblyNames[idx]); mergeVertex.AddArgument(v.dynamicManager.classNames[idx]); mergeVertex.AddArgument(v.dynamicManager.methodNames[idx]); idx++; dynamicMerge.SetInternalVertex(mergeVertex); dynamicMerge.SetGroupingSettings(0, 0); dynamicMerge.SetPodGroupingSettings(lowThreshold, highThreshold); dynamicMerge.SetDataGroupingSettings(lowDataThreshold, highDataThreshold, maxSingleDataThreshold); dynamicMerge.SetMaxAggregationLevel(v.dynamicManager.aggregationLevels); } if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR) { newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, dynamicMerge); } else { string name = v.dynamicManager.methodNames[idx]; DrManagerBase newStage = new DrManagerBase(app.GetGraph(), name); DrActiveVertex distributeVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate); distributeVertex.AddArgument(stdVertexName); distributeVertex.AddArgument(v.dynamicManager.assemblyNames[idx]); distributeVertex.AddArgument(v.dynamicManager.classNames[idx]); distributeVertex.AddArgument(v.dynamicManager.methodNames[idx]); idx++; DrDynamicDistributionManager dynamicHashDistribute = new DrDynamicDistributionManager(distributeVertex, dynamicMerge); dynamicHashDistribute.SetDataPerVertex(highDataThreshold * 2); // 2GB newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, dynamicHashDistribute); } } else if (v.dynamicManager.type == DynamicManager.Type.RANGEDISTRIBUTOR) { DrStageManager splitManager = graphStageMap[v.dynamicManager.splitVertexId].stageManager; DrDynamicRangeDistributionManager drdm = new DrDynamicRangeDistributionManager(splitManager, v.dynamicManager.sampleRate); drdm.SetDataPerVertex(highDataThreshold * 2); // 2GB newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, drdm); } else if (v.dynamicManager.type == DynamicManager.Type.BROADCAST) { // the copy vertex int bcastNumber = 0; string nameString = String.Format("CP__{0}", bcastNumber++); DrManagerBase newStage = new DrManagerBase(app.GetGraph(), nameString); DrActiveVertex copyVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate); copyVertex.AddArgument(cpyVertexName); DrDynamicBroadcastManager bcast = new DrDynamicBroadcastManager(copyVertex); newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, bcast); } else if (v.dynamicManager.type != DynamicManager.Type.NONE) { DryadLogger.LogWarning("Build Graph From Query", "Dynamic manager type {0} not supported yet", v.dynamicManager.type); } } } // // Add all the edges // DryadLogger.LogInformation("Build Graph From Query", "Adding edges"); foreach (KeyValuePair <int, GraphStageInfo> kvp in graphStageMap) { AddEdges(kvp.Value, graphStageMap); } // // Register the actual created vertices with the graph // MaterializeToManagers(graphStageMap); }