/// <summary> /// Initialize the job directory for vertex execution /// </summary> /// <param name="resources">list of DryadLINQ-requested resources</param> /// <returns>success/failure</returns> public static bool InitializeForJobExecution(string resources) { try { // // Update list of known local binaries if needed // InitializeBinFileList(); ProcessPathHelper.CreateUserWorkingDirectory(); Directory.CreateDirectory(ProcessPathHelper.JobPath); // // copy any files that already live locally and may be needed for the job // bool success = CopyLocalBinaries(); // // copy any user-specified files that haven't already been copied // success &= CopyStagedJobResources(resources); return(success); } catch (Exception e) { // // Write out any errors and return false on exception // Console.Error.WriteLine("[ExecutionHelper.InitializeForJobExecution] Exception: {0}", e.Message); Console.Error.WriteLine(e.StackTrace); return(false); } }
/// <summary> /// Create working directory for vertex /// </summary> /// <param name="id"></param> /// <returns></returns> public static bool InitializeForProcessExecution(int id, string resources) { try { Directory.CreateDirectory(ProcessPathHelper.ProcessPath(id)); Console.Error.WriteLine("Created directory: " + ProcessPathHelper.ProcessPath(id)); return(true); } catch (Exception e) { Console.Error.WriteLine("[ExecutionHelper.InitializeForProcessExecution] Exception: {0}", e.Message); Console.Error.WriteLine(e.StackTrace); return(false); } }
/// <summary> /// The main entry point for the application. /// </summary> private static int Main(string[] args) { // // Try to create working directory. Fail vertex service if unable to do so. // bool createdJobDir = false; int retryCount = 0; do { try { ProcessPathHelper.CreateUserWorkingDirectory(); Directory.CreateDirectory(ProcessPathHelper.JobPath); createdJobDir = true; } catch (Exception ex) { Console.Error.WriteLine("Failed to create working directory, {0}. Error: {1}.", ProcessPathHelper.JobPath, ex.ToString()); retryCount++; } } while (retryCount < numRetries && !createdJobDir); if (!createdJobDir) { Console.Error.WriteLine("Vertex service cannot proceed because working directory could not be created."); return(1); } // // Get Task ID from environment // int taskId; if (Int32.TryParse(Environment.GetEnvironmentVariable("CCP_TASKID"), out taskId) == false) { Console.Error.WriteLine("Program.Main", "Failed to read CCP_TASKID from environment"); return(1); } // // Initialize tracing subsystem // string traceFile = Path.Combine(ProcessPathHelper.JobPath, String.Format("VertexServiceTrace_{0}.txt", taskId)); DryadLogger.Start(traceFile); // // Initialize scheduler helper of the correct type // ISchedulerHelper schedulerHelper; try { schedulerHelper = SchedulerHelperFactory.GetInstance(); } catch (Exception ex) { DryadLogger.LogCritical(0, ex, "Failed to get scheduler helper"); DryadLogger.Stop(); Console.Error.WriteLine("Failed to contact HPC scheduler. See log for details."); return(1); } // // Step 1 of the address configuration procedure: Create a URI to serve as the base address. // string strAddress = schedulerHelper.GetVertexServiceBaseAddress("localhost", taskId); Uri baseAddress = new Uri(strAddress); // // Step 2 of the hosting procedure: Create ServiceHost // ServiceHost selfHost = new ServiceHost(typeof(VertexService), baseAddress); try { // // Get the service binding // NetTcpBinding binding = schedulerHelper.GetVertexServiceBinding(); // // Step 3 of the hosting procedure: Add service endpoints. // ServiceEndpoint vertexEndpoint = selfHost.AddServiceEndpoint(typeof(IDryadVertexService), binding, Constants.vertexServiceName); DryadLogger.LogInformation("Initialize vertex service", "listening on address {0}", vertexEndpoint.Address.ToString()); // // Step 4 of hosting procedure : Add a security manager // TODO: Fix this for local scheduler and / or Azure scheduler when supported // selfHost.Authorization.ServiceAuthorizationManager = new DryadVertexServiceAuthorizationManager(); // Step 5 of the hosting procedure: Start (and then stop) the service. selfHost.Open(); Console.WriteLine("Vertex Service up and waiting for commands"); // Wait for the shutdown event to be set. VertexService.shutdownEvent.WaitOne(-1, true); // Check vertex service shutdown condition if (VertexService.internalShutdown) { string errorMsg = string.Format("Vertex Service Task unable to continue after critical error in initialization or communication: {0}", VertexService.ShutdownReason.ToString()); Console.WriteLine(errorMsg); DryadLogger.LogCritical(0, new Exception(errorMsg)); DryadLogger.Stop(); try { selfHost.Abort(); } catch { } return(1); } // Close the ServiceHostBase to shutdown the service. selfHost.Close(); } catch (CommunicationException ce) { // // Report any errors and fail task // DryadLogger.LogCritical(0, ce, "A communication exception occurred"); DryadLogger.Stop(); try { selfHost.Abort(); } catch { } Console.Error.WriteLine("CommunicationException occured, aborting vertex service. See log for details."); return(1); } catch (Exception ex) { // // Report any errors and fail task // DryadLogger.LogCritical(0, ex, "An exception occurred"); DryadLogger.Stop(); try { selfHost.Abort(); } catch { } Console.Error.WriteLine("An exception occured, aborting vertex service. See log for details."); return(1); } DryadLogger.LogInformation("Vertex Service", "Shut down cleanly"); DryadLogger.Stop(); return(0); }
/// <summary> /// Asynchronously called on start command /// </summary> /// <param name="obj"></param> void StartProcessThreadProc(Object obj) { ManualResetEvent serviceInitializedEvent = obj as ManualResetEvent; bool started = false; try { // // Wait for service initialization // serviceInitializedEvent.WaitOne(); if (ExecutionHelper.InitializeForProcessExecution(dryadProcessId, Environment.GetEnvironmentVariable("XC_RESOURCEFILES"))) { // // Vertex working directory configured successfully, start the vertex host // environment.Add(Constants.vertexSvcLocalAddrEnvVar, localAddress); ProcessStartInfo startInfo = new ProcessStartInfo(); startInfo.CreateNoWindow = true; startInfo.UseShellExecute = false; startInfo.WorkingDirectory = ProcessPathHelper.ProcessPath(dryadProcessId); //YARN Debugging //var procEnvVarKeys = startInfo.EnvironmentVariables.Keys; //foreach (string key in procEnvVarKeys) //{ // DryadLogger.LogInformation("StartProcess", "key: '{0}' value: '{1}'", key, startInfo.EnvironmentVariables[key]); //} string[] args = commandLine.Split(' '); string arg = ""; for (int i = 1; i < args.Length; i++) { arg += args[i] + " "; } // // Use either FQ path or path relative to job path // if (Path.IsPathRooted(args[0])) { startInfo.FileName = args[0]; } else { startInfo.FileName = Path.Combine(ProcessPathHelper.JobPath, args[0]); } DryadLogger.LogInformation("StartProcess", "FileName: '{0}'", startInfo.FileName); // // Add environment variable to vertex host process // startInfo.Arguments = arg; foreach (DictionaryEntry entry in environment) { string key = entry.Key.ToString(); if (key == null || startInfo.EnvironmentVariables.ContainsKey(key)) { DryadLogger.LogInformation("StartProcess", "Attempting to add existing key '{0}' with value '{1}'", entry.Key, entry.Value); } else { startInfo.EnvironmentVariables.Add(key, entry.Value.ToString()); } } lock (syncRoot) { // // After taking lock, start the vertex host process and set up exited event handler // if (cancelled) { // If we've already been canceled, don't start the process DryadLogger.LogInformation("Process start", "Not starting process {0} due to receipt of cancellation", DryadId); return; } else { systemProcess = new Process(); systemProcess.StartInfo = startInfo; systemProcess.EnableRaisingEvents = true; systemProcess.Exited += new EventHandler(Process_Exited); Console.WriteLine("Process start - Vertex host process starting"); started = systemProcess.Start(); Console.WriteLine("Process start - Vertex host process started"); if (started) { DryadLogger.LogInformation("Process start", "Vertex host process started"); state = ProcessState.Running; } else { DryadLogger.LogError(0, null, "Vertex host process failed to start"); } } } } else { DryadLogger.LogError(0, null, "Initialization failed"); } } catch (Exception e) { DryadLogger.LogError(0, e, "Error starting vertex"); } if (started) { // // Notify Graph Manager that process started if successful // bool success = ReplyDispatcher.FireStateChange(this.graphManagerReplyUri, this.dryadProcessId, ProcessState.Running); if (!success) { // // Graph manager doesn't know we started and we have no way to tell it, so it's // best to just fail the vertex service task and let the job manager inform the graph manager // VertexService.Surrender(new Exception("Unable to communicate with graph manager.")); } } else { // // Otherwise, notify GM that process has failed // lock (syncRoot) { // If we've already been canceled, we don't need to change state or record the initialization failure if (!cancelled) { state = ProcessState.Completed; this.failed = true; exitCode = unchecked ((int)Constants.DrError_VertexInitialization); // DryadError_VertexInitialization } } if (failed) // This also means we weren't canceled { // Notify the Graph Manager that the process failed to start Process_Exited(this, null); } } // // Make sure process start event is set // processStartEvent.Set(); }