Пример #1
0
        static void ExceptionHandler(object sender, UnhandledExceptionEventArgs args)
        {
            int       result = unchecked ((int)LinqToDryadException.E_FAIL);
            Exception e      = args.ExceptionObject as Exception;

            if (e != null)
            {
                result = System.Runtime.InteropServices.Marshal.GetHRForException(e);
                DryadLogger.LogCritical(0, e);
            }
            DebugHelper.StopLogging(result);

            if (Debugger.IsAttached)
            {
                Debugger.Break();
            }
            else
            {
                DrLogging.WriteMiniDump();
            }

            // We need to Exit, since other threads in the GM
            // are likely to still be running.
            Environment.Exit(result);
        }
Пример #2
0
        public bool ParseQueryXml(string queryPlanFileName, Query query)
        {
            XmlNode     version      = null;
            XmlDocument queryPlanDoc = new XmlDocument();

            //
            // Load query plan document
            //
            try
            {
                queryPlanDoc.Load(queryPlanFileName);
            }
            catch (Exception e)
            {
                DryadLogger.LogCritical(0, e, "Failed to load query plan: {0}", queryPlanFileName);
                return(false);
            }

            //
            // Get DryadLinqVersion - absence used to indicate Argentia query plan
            //
            try
            {
                version = queryPlanDoc.DocumentElement.SelectSingleNode("DryadLinqVersion");
            }
            catch (System.Xml.XPath.XPathException e)
            {
                DryadLogger.LogCritical(0, e, "Failed to select node DryadLinqVersion from query plan: {0}", queryPlanFileName);
                return(false);
            }

            if (version == null)
            {
                DryadLogger.LogCritical(0, null, "Missing element 'DryadLinqVersion' in query plan: {0}", queryPlanFileName);
                return(false);
            }

            //
            // Parse query plan XML doc into Query
            //
            try
            {
                ParseQueryXmlLinqToDryad(queryPlanDoc, query);
            }
            catch (Exception e)
            {
                DryadLogger.LogCritical(0, e, "Failed to parse query plan: {0}", queryPlanFileName);
                return(false);
            }
Пример #3
0
        static int Main(string[] args)
        {
            Console.WriteLine("{0} starting up in {1}", Environment.CommandLine, Environment.CurrentDirectory);

            bool          waitForDebugger = false;
            List <string> tempArgs        = new List <string>();

            // Record start time so we can report it once logging has been initialized
            DateTime startTime = DateTime.Now.ToLocalTime();

            // Set unhandled exception handler to catch anything thrown from
            // Microsoft.Hpc.Query.GraphManager.dll
            AppDomain currentDomain = AppDomain.CurrentDomain;

            currentDomain.UnhandledException += new UnhandledExceptionEventHandler(ExceptionHandler);


            //
            // Add executable name to beginning of args
            // to make graph manager libraries happy
            //
            tempArgs.Add("YarnQueryGraphManager");
            foreach (string arg in args)
            {
                if (String.Compare(arg, "--break", StringComparison.OrdinalIgnoreCase) == 0)
                {
                    waitForDebugger = true;
                }
                else
                {
                    tempArgs.Add(arg);
                }
            }
            args = tempArgs.ToArray();

            if (waitForDebugger)
            {
                DebugHelper.WaitForDebugger();
            }

            /* Yarn removes the need for this
             *
             * //
             * // Create job directory and copy resources
             * //
             * string resources = Environment.GetEnvironmentVariable("XC_RESOURCEFILES");
             * if (ExecutionHelper.InitializeForJobExecution(resources) == false)
             * {
             *  return 1;
             * }
             *
             * //
             * // Set current directory to working directory
             * //
             * Directory.SetCurrentDirectory(ProcessPathHelper.JobPath);
             */
            //
            // Configure tracing
            //
            DebugHelper.InitializeLogging(startTime);

            // Ensure that there is a jvm.dll on the path.
            string pathString = Environment.GetEnvironmentVariable("PATH");
            var    pathDirs   = pathString.Split(';');
            bool   found      = false;

            foreach (var dir in pathDirs)
            {
                string targetFile = Path.Combine(dir, "jvm.dll");
                if (File.Exists(targetFile))
                {
                    found = true;
                    break;
                }
            }
            if (!found)
            {
                string javaHome = Environment.GetEnvironmentVariable("JAVA_HOME");
                if (String.IsNullOrEmpty(javaHome))
                {
                    throw new ApplicationException("DryadLINQ requires the JAVA_HOME environment variable to be set or the jvm.dll to be on the path.");
                }
                var jvmPath = ";" + Path.Combine(javaHome, "jre", "bin", "server");
                pathString += jvmPath;
                Environment.SetEnvironmentVariable("PATH", pathString);
            }


            //
            // Run the Graph Manager
            //
            int retCode = 0;

            try
            {
                LinqToDryadJM jm = new LinqToDryadJM();
                retCode = jm.Run(args);
            }
            catch (Exception e)
            {
                retCode = System.Runtime.InteropServices.Marshal.GetHRForException(e);
                if (retCode == 0)
                {
                    DryadLogger.LogCritical(0, e);
                    retCode = unchecked ((int)LinqToDryadException.E_FAIL);
                }
                else
                {
                    DryadLogger.LogCritical(retCode, e);
                }
                System.Threading.Thread.Sleep(10 * 60 * 1000);
            }

            DebugHelper.StopLogging(retCode);

            // NOTE: We don't want to log critical errors twice, so here we're assuming that
            // if the GM exited "gracefully" and returned an error code instead of throwing
            // an exception, that it has already logged the error.
            if (retCode != 0)
            {
                // If the Graph Manager already started executing, we need to exit the process.
                // Exiting the thread (returning from Main) will not necessarily cause the GM's
                // worker threads to exit

                // TODO: Consider deleting temp output stream from DSC
                // requires that we have access to the URI at this point, though
                Environment.Exit(retCode);
            }

            //
            // Cleanup all vertex tasks in case any became
            // unreachable.
            //

            return(retCode);
        }
Пример #4
0
        //
        // Main Dryad LINQ execution stuff
        //
        public int ExecLinqToDryad(string[] args)
        {
            //
            // must be at least two arguments (program name and query XML file name)
            //
            if (args.Length < 2)
            {
                DryadLogger.LogCritical(0, null, "Must provide at least query XML file name.");
                return(-1);
            }

            //
            // break if --break is included in arguments (and eliminate it, as it is not known downstream)
            //
            if (ConsumeSingleArgument("--break", ref args))
            {
                DebugHelper.WaitForDebugger();
            }

            //
            // parse the XML input, producing a DryadLINQ Query
            //
            Query           query  = new Query();
            QueryPlanParser parser = new QueryPlanParser();

            if (!parser.ParseQueryXml(args[1], query))
            {
                DryadLogger.LogCritical(0, null, "Invalid query plan");
                return(-1);
            }

            //
            // build internal app arguments
            //
            List <string> internalArgs = new List <string>();

            //
            // add the XmlExecHost args to the internal app arguments
            //
            foreach (string xmlExecHostArg in query.xmlExecHostArgs)

            {
                if (xmlExecHostArg == "--break")
                {
                    DebugHelper.WaitForDebugger();
                }
                else
                {
                    internalArgs.Add(xmlExecHostArg);
                }
            }

            //
            // combine internal arguments with any additional arguments received on the command line
            // don't include argv[0] and argv[1] (program name and query XML file name)
            //

            int internalArgc = (int)internalArgs.Count;
            int externalArgc = args.Length - 2;          // don't include argv[0] and argv[1]
            int combinedArgc = internalArgc + externalArgc;

            string[] combinedArgv = new string[combinedArgc];

            string msg = "";

            // internal arguments first
            for (int i = 0; i < internalArgc; i++)
            {
                combinedArgv[i] = internalArgs[i];
                msg            += String.Format("{0} ", combinedArgv[i]);
            }

            // then external arguments
            for (int i = 0; i < externalArgc; i++)
            {
                combinedArgv[i + internalArgc] = args[i + 2]; // don't include argv[0] and argv[1]

                msg += String.Format("{0} ", combinedArgv[i + internalArgc]);
            }
            DryadLogger.LogInformation(null, "Arguments: {0}", msg);

            string jobClass    = "DryadLINQ";
            string dryadBinDir = Environment.GetEnvironmentVariable("DRYAD_HOME");

            if (String.IsNullOrEmpty(dryadBinDir))
            {
                throw new ApplicationException("DryadLINQ requires the DRYAD_HOME environment variable to be set to the Dryad binary folder.");
            }
            string exeName = Path.Combine(dryadBinDir, "VertexHost.exe");

            // create app and run it
            //
            DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication);

            DrArtemisLegacyReporter reporter = new DrArtemisLegacyReporter();

            p.m_defaultProcessTemplate.GetListenerList().Add(reporter);
            p.m_defaultVertexTemplate.GetListenerList().Add(reporter);
            p.m_topologyReporter            = reporter;
            p.m_intermediateCompressionMode = query.intermediateDataCompression;
            DrGraphExecutor graphExecutor = new DrGraphExecutor();
            DrGraph         graph         = graphExecutor.Initialize(p);

            if (graph == null)
            {
                DryadLogger.LogCritical(0, null, "Failed to initialize Graph Executor");
                return(-1);
            }
            DryadLINQApp app = new DryadLINQApp(graph);

            // Initialize with arguments
            app.SetXmlFileName(args[1]);
            if (!app.ParseCommandLineFlags(combinedArgv))
            {
                DryadLogger.LogCritical(0, null, "Bad command-line options");
                return(-1);
            }
            // Build graph from query plan
            GraphBuilder builder = new GraphBuilder();

            builder.BuildGraphFromQuery(app, query);

            // Run the app
            DryadLogger.LogInformation(null, "Running the app");

            graphExecutor.Run();
            DrError exitStatus = graphExecutor.Join();

            DryadLogger.LogInformation(null, "Finished running the app");

            if (exitStatus == null || exitStatus.m_code == 0)
            {
                FinalizeExecution(query, graph);
                DryadLogger.LogInformation(null, "Application completed successfully.");
                return(0);
            }
            else
            {
                DryadLogger.LogCritical(exitStatus.m_code, null, "Application failed with error code 0x{0:X8}.\n", exitStatus.m_code);
                return(exitStatus.m_code);
            }
        }
Пример #5
0
        public bool ParseCommandLineFlags(string[] args)
        {
            bool retVal = true;

            for (int index = 0; index < args.Length; index++)
            {
                string            arg    = args[index].Substring(args[index].IndexOf("-") + 1);
                OptionDescription option = null;
                m_optionMap.TryGetValue(arg, out option);
                if (option == null)
                {
                    retVal = false;
                    break;
                }
                int optionIndex = option.m_optionIndex;

                switch (optionIndex)
                {
                case (int)DryadLINQAppOptions.BDJAO_AMD64:
                case (int)DryadLINQAppOptions.BDJAO_I386:
                case (int)DryadLINQAppOptions.BDJAO_Retail:
                case (int)DryadLINQAppOptions.BDJAO_Debug:
                    break;

                case (int)DryadLINQAppOptions.DNAO_MaxAggregateInputs:
                    if ((index + 1) >= args.Length)
                    {
                        DryadLogger.LogCritical(0, null, "The argument for option '{0}' was missing.\n", args[index]);
                        retVal = false;
                    }
                    else
                    {
                        int maxInputs;
                        if (!Int32.TryParse(args[index + 1], out maxInputs))
                        {
                            DryadLogger.LogCritical(0, null, "The argument '{0}' for option '{1}' could not be parsed as an integer.\n", args[index + 1], args[index]);
                            retVal = false;
                        }
                        else
                        {
                            m_maxAggregateInputs = maxInputs;
                            index++;
                        }
                    }
                    break;

                case (int)DryadLINQAppOptions.DNAO_MaxAggregateFilterInputs:
                    if ((index + 1) >= args.Length)
                    {
                        DryadLogger.LogCritical(0, null, "The argument for option '{0}' was missing.\n", args[index]);
                        retVal = false;
                    }
                    else
                    {
                        int maxInputs;
                        if (!Int32.TryParse(args[index + 1], out maxInputs))
                        {
                            DryadLogger.LogCritical(0, null, "The argument '{0}' for option '{1}' could not be parsed as an integer.\n", args[index + 1], args[index]);
                            retVal = false;
                        }
                        else
                        {
                            m_maxAggregateFilterInputs = maxInputs;
                            index++;
                        }
                    }
                    break;


                case (int)DryadLINQAppOptions.DNAO_AggregateThreshold:
                    if ((index + 1) >= args.Length)
                    {
                        DryadLogger.LogCritical(0, null, "The argument for option '{0}' was missing.\n", args[index]);
                        retVal = false;
                    }
                    else
                    {
                        UInt64 threshold;
                        if (!UInt64.TryParse(args[index + 1], out threshold))
                        {
                            DryadLogger.LogCritical(0, null, "The argument '{0}' for option '{1}' could not be parsed as a UIN64.\n", args[index + 1], args[index]);
                            retVal = false;
                        }
                        else
                        {
                            m_aggregateThreshold = threshold;
                            index++;
                        }
                    }
                    break;

                case (int)DryadLINQAppOptions.DNAO_NoClusterAffinity:
                    m_clusterAffinity = false;
                    break;

                default:
                    DryadLogger.LogCritical(0, null, "Unknown command-line option {0}\n", optionIndex);
                    retVal = false;
                    break;
                }
            }

            if (!retVal)
            {
                PrintUsage(args[0]);
            }
            return(retVal);
        }