/// <summary>
/// Initialize the job directory for vertex execution
/// </summary>
/// <param name="resources">list of DryadLINQ-requested resources</param>
/// <returns>success/failure</returns>
public static bool InitializeForJobExecution(string resources)
{
try
{
//
// Update list of known local binaries if needed
//
InitializeBinFileList();
ProcessPathHelper.CreateUserWorkingDirectory();
Directory.CreateDirectory(ProcessPathHelper.JobPath);
//
// copy any files that already live locally and may be needed for the job
//
bool success = CopyLocalBinaries();
//
// copy any user-specified files that haven't already been copied
//
success &= CopyStagedJobResources(resources);
return(success);
}
catch (Exception e)
{
//
// Write out any errors and return false on exception
//
Console.Error.WriteLine("[ExecutionHelper.InitializeForJobExecution] Exception: {0}", e.Message);
Console.Error.WriteLine(e.StackTrace);
return(false);
}
}
/// <summary>
/// Create working directory for vertex
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public static bool InitializeForProcessExecution(int id, string resources)
{
try
{
Directory.CreateDirectory(ProcessPathHelper.ProcessPath(id));
Console.Error.WriteLine("Created directory: " + ProcessPathHelper.ProcessPath(id));
return(true);
}
catch (Exception e)
{
Console.Error.WriteLine("[ExecutionHelper.InitializeForProcessExecution] Exception: {0}", e.Message);
Console.Error.WriteLine(e.StackTrace);
return(false);
}
}
/// <summary>
/// The main entry point for the application.
/// </summary>
private static int Main(string[] args)
{
//
// Try to create working directory. Fail vertex service if unable to do so.
//
bool createdJobDir = false;
int retryCount = 0;
do
{
try
{
ProcessPathHelper.CreateUserWorkingDirectory();
Directory.CreateDirectory(ProcessPathHelper.JobPath);
createdJobDir = true;
}
catch (Exception ex)
{
Console.Error.WriteLine("Failed to create working directory, {0}. Error: {1}.", ProcessPathHelper.JobPath, ex.ToString());
retryCount++;
}
} while (retryCount < numRetries && !createdJobDir);
if (!createdJobDir)
{
Console.Error.WriteLine("Vertex service cannot proceed because working directory could not be created.");
return(1);
}
//
// Get Task ID from environment
//
int taskId;
if (Int32.TryParse(Environment.GetEnvironmentVariable("CCP_TASKID"), out taskId) == false)
{
Console.Error.WriteLine("Program.Main", "Failed to read CCP_TASKID from environment");
return(1);
}
//
// Initialize tracing subsystem
//
string traceFile = Path.Combine(ProcessPathHelper.JobPath, String.Format("VertexServiceTrace_{0}.txt", taskId));
DryadLogger.Start(traceFile);
//
// Initialize scheduler helper of the correct type
//
ISchedulerHelper schedulerHelper;
try
{
schedulerHelper = SchedulerHelperFactory.GetInstance();
}
catch (Exception ex)
{
DryadLogger.LogCritical(0, ex, "Failed to get scheduler helper");
DryadLogger.Stop();
Console.Error.WriteLine("Failed to contact HPC scheduler. See log for details.");
return(1);
}
//
// Step 1 of the address configuration procedure: Create a URI to serve as the base address.
//
string strAddress = schedulerHelper.GetVertexServiceBaseAddress("localhost", taskId);
Uri baseAddress = new Uri(strAddress);
//
// Step 2 of the hosting procedure: Create ServiceHost
//
ServiceHost selfHost = new ServiceHost(typeof(VertexService), baseAddress);
try
{
//
// Get the service binding
//
NetTcpBinding binding = schedulerHelper.GetVertexServiceBinding();
//
// Step 3 of the hosting procedure: Add service endpoints.
//
ServiceEndpoint vertexEndpoint = selfHost.AddServiceEndpoint(typeof(IDryadVertexService), binding, Constants.vertexServiceName);
DryadLogger.LogInformation("Initialize vertex service", "listening on address {0}", vertexEndpoint.Address.ToString());
//
// Step 4 of hosting procedure : Add a security manager
// TODO: Fix this for local scheduler and / or Azure scheduler when supported
//
selfHost.Authorization.ServiceAuthorizationManager = new DryadVertexServiceAuthorizationManager();
// Step 5 of the hosting procedure: Start (and then stop) the service.
selfHost.Open();
Console.WriteLine("Vertex Service up and waiting for commands");
// Wait for the shutdown event to be set.
VertexService.shutdownEvent.WaitOne(-1, true);
// Check vertex service shutdown condition
if (VertexService.internalShutdown)
{
string errorMsg = string.Format("Vertex Service Task unable to continue after critical error in initialization or communication: {0}", VertexService.ShutdownReason.ToString());
Console.WriteLine(errorMsg);
DryadLogger.LogCritical(0, new Exception(errorMsg));
DryadLogger.Stop();
try
{
selfHost.Abort();
}
catch
{
}
return(1);
}
// Close the ServiceHostBase to shutdown the service.
selfHost.Close();
}
catch (CommunicationException ce)
{
//
// Report any errors and fail task
//
DryadLogger.LogCritical(0, ce, "A communication exception occurred");
DryadLogger.Stop();
try
{
selfHost.Abort();
}
catch
{
}
Console.Error.WriteLine("CommunicationException occured, aborting vertex service. See log for details.");
return(1);
}
catch (Exception ex)
{
//
// Report any errors and fail task
//
DryadLogger.LogCritical(0, ex, "An exception occurred");
DryadLogger.Stop();
try
{
selfHost.Abort();
}
catch
{
}
Console.Error.WriteLine("An exception occured, aborting vertex service. See log for details.");
return(1);
}
DryadLogger.LogInformation("Vertex Service", "Shut down cleanly");
DryadLogger.Stop();
return(0);
}
/// <summary>
/// Asynchronously called on start command
/// </summary>
/// <param name="obj"></param>
void StartProcessThreadProc(Object obj)
{
ManualResetEvent serviceInitializedEvent = obj as ManualResetEvent;
bool started = false;
try
{
//
// Wait for service initialization
//
serviceInitializedEvent.WaitOne();
if (ExecutionHelper.InitializeForProcessExecution(dryadProcessId, Environment.GetEnvironmentVariable("XC_RESOURCEFILES")))
{
//
// Vertex working directory configured successfully, start the vertex host
//
environment.Add(Constants.vertexSvcLocalAddrEnvVar, localAddress);
ProcessStartInfo startInfo = new ProcessStartInfo();
startInfo.CreateNoWindow = true;
startInfo.UseShellExecute = false;
startInfo.WorkingDirectory = ProcessPathHelper.ProcessPath(dryadProcessId);
//YARN Debugging
//var procEnvVarKeys = startInfo.EnvironmentVariables.Keys;
//foreach (string key in procEnvVarKeys)
//{
// DryadLogger.LogInformation("StartProcess", "key: '{0}' value: '{1}'", key, startInfo.EnvironmentVariables[key]);
//}
string[] args = commandLine.Split(' ');
string arg = "";
for (int i = 1; i < args.Length; i++)
{
arg += args[i] + " ";
}
//
// Use either FQ path or path relative to job path
//
if (Path.IsPathRooted(args[0]))
{
startInfo.FileName = args[0];
}
else
{
startInfo.FileName = Path.Combine(ProcessPathHelper.JobPath, args[0]);
}
DryadLogger.LogInformation("StartProcess", "FileName: '{0}'", startInfo.FileName);
//
// Add environment variable to vertex host process
//
startInfo.Arguments = arg;
foreach (DictionaryEntry entry in environment)
{
string key = entry.Key.ToString();
if (key == null || startInfo.EnvironmentVariables.ContainsKey(key))
{
DryadLogger.LogInformation("StartProcess", "Attempting to add existing key '{0}' with value '{1}'",
entry.Key, entry.Value);
}
else
{
startInfo.EnvironmentVariables.Add(key, entry.Value.ToString());
}
}
lock (syncRoot)
{
//
// After taking lock, start the vertex host process and set up exited event handler
//
if (cancelled)
{
// If we've already been canceled, don't start the process
DryadLogger.LogInformation("Process start", "Not starting process {0} due to receipt of cancellation", DryadId);
return;
}
else
{
systemProcess = new Process();
systemProcess.StartInfo = startInfo;
systemProcess.EnableRaisingEvents = true;
systemProcess.Exited += new EventHandler(Process_Exited);
Console.WriteLine("Process start - Vertex host process starting");
started = systemProcess.Start();
Console.WriteLine("Process start - Vertex host process started");
if (started)
{
DryadLogger.LogInformation("Process start", "Vertex host process started");
state = ProcessState.Running;
}
else
{
DryadLogger.LogError(0, null, "Vertex host process failed to start");
}
}
}
}
else
{
DryadLogger.LogError(0, null, "Initialization failed");
}
}
catch (Exception e)
{
DryadLogger.LogError(0, e, "Error starting vertex");
}
if (started)
{
//
// Notify Graph Manager that process started if successful
//
bool success = ReplyDispatcher.FireStateChange(this.graphManagerReplyUri, this.dryadProcessId, ProcessState.Running);
if (!success)
{
//
// Graph manager doesn't know we started and we have no way to tell it, so it's
// best to just fail the vertex service task and let the job manager inform the graph manager
//
VertexService.Surrender(new Exception("Unable to communicate with graph manager."));
}
}
else
{
//
// Otherwise, notify GM that process has failed
//
lock (syncRoot)
{
// If we've already been canceled, we don't need to change state or record the initialization failure
if (!cancelled)
{
state = ProcessState.Completed;
this.failed = true;
exitCode = unchecked ((int)Constants.DrError_VertexInitialization); // DryadError_VertexInitialization
}
}
if (failed) // This also means we weren't canceled
{
// Notify the Graph Manager that the process failed to start
Process_Exited(this, null);
}
}
//
// Make sure process start event is set
//
processStartEvent.Set();
}
