public static void Run(HashSet <string> appDlls, IConfiguration driverBridgeConfig, DriverSubmissionSettings driverSubmissionSettings, string reefJar = Constants.JavaBridgeJarFileName, string runCommand = "run.cmd", string clrFolder = ".", string className = Constants.BridgeLaunchClass)
{
using (LOGGER.LogFunction("ClrHandlerHelper::Run"))
{
if (driverSubmissionSettings.Submit)
{
ClrHandlerHelper.CopyDllsToAppDirectory(appDlls);
UpdateJarFileWithAssemblies(reefJar);
}
using (LOGGER.LogScope("ClrHandlerHelper::serialize driverBridgeConfig to clrRuntimeConfigFile"))
{
string clrRuntimeConfigFile = Path.Combine(clrFolder, Constants.DriverBridgeConfiguration);
new AvroConfigurationSerializer().ToFile(driverBridgeConfig, clrRuntimeConfigFile);
LOGGER.Log(Level.Info, "CLR driver bridge configurations written to " + clrRuntimeConfigFile);
}
ProcessStartInfo startInfo = new ProcessStartInfo();
if (driverSubmissionSettings.RunOnYarn)
{
startInfo.FileName = runCommand;
startInfo.Arguments = className + " " + clrFolder +
driverSubmissionSettings.ToComamndLineArguments();
}
else
{
startInfo.FileName = GetJavaBinary();
string loggingPrefix = string.Empty;
if (driverSubmissionSettings.JavaLogLevel == JavaLoggingSetting.VERBOSE_TO_CLR)
{
loggingPrefix = Constants.JavaToCLRLoggingConfig + " ";
}
else if (driverSubmissionSettings.JavaLogLevel == JavaLoggingSetting.VERBOSE)
{
loggingPrefix = Constants.JavaVerboseLoggingConfig + " ";
}
startInfo.Arguments = loggingPrefix + @"-classpath " + reefJar + " " + Constants.BridgeLaunchClass +
" " + clrFolder + " " + driverSubmissionSettings.ToComamndLineArguments();
}
startInfo.RedirectStandardOutput = true;
startInfo.UseShellExecute = false;
startInfo.CreateNoWindow = false;
LOGGER.Log(Level.Info, "Executing\r\n" + startInfo.FileName + "\r\n" + startInfo.Arguments);
using (Process process = Process.Start(startInfo))
{
process.WaitForExit();
}
}
}
public static void UpdateJarFileWithAssemblies(string reefJar)
{
using (LOGGER.LogFunction("ClrHandlerHelper::UpdateJarFileWithAssemblies"))
{
string assembliesList = ClrHandlerHelper.GetAssembliesListForReefDriverApp();
if (!File.Exists(reefJar))
{
throw new InvalidOperationException("cannot find reef jar file: " + reefJar);
}
ProcessStartInfo startInfo = new ProcessStartInfo()
{
FileName = GetJarBinary(),
Arguments = @"uf " + reefJar + " " + assembliesList,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
LOGGER.Log(Level.Info, "updating jar file with \r\n" + startInfo.FileName + "\r\n" + startInfo.Arguments);
using (Process process = Process.Start(startInfo))
{
StreamReader outReader = process.StandardOutput;
StreamReader errorReader = process.StandardError;
string output = outReader.ReadToEnd();
string error = errorReader.ReadToEnd();
process.WaitForExit();
if (process.ExitCode != 0)
{
throw new InvalidOperationException("Failed to update jar file with stdout :" + output +
"and stderr:" + error);
}
}
LOGGER.Log(Level.Info, "jar file updated.");
}
}
public BridgeHandlerManager Subscribe()
{
var bridgeHandlerManager = new BridgeHandlerManager();
// subscribe to Allocated Evaluator
foreach (var handler in _allocatedEvaluatorHandlers)
{
_allocatedEvaluatorSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to IAllocatedEvaluator handler: " + handler);
}
bridgeHandlerManager.AllocatedEvaluatorHandler = ClrHandlerHelper.CreateHandler(_allocatedEvaluatorSubscriber);
// subscribe to TaskMessage
foreach (var handler in _taskMessageHandlers)
{
_taskMessageSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to ITaskMessage handler: " + handler);
}
bridgeHandlerManager.TaskMessageHandler = ClrHandlerHelper.CreateHandler(_taskMessageSubscriber);
// subscribe to Active Context
foreach (var handler in _activeContextHandlers)
{
_activeContextSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to IActiveContext handler: " + handler);
}
bridgeHandlerManager.ActiveContextHandler = ClrHandlerHelper.CreateHandler(_activeContextSubscriber);
// subscribe to Failed Task
foreach (var handler in _failedTaskHandlers)
{
_failedTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to IFailedTask handler: " + handler);
}
bridgeHandlerManager.FailedTaskHandler = ClrHandlerHelper.CreateHandler(_failedTaskSubscriber);
// subscribe to Running Task
foreach (var handler in _runningTaskHandlers)
{
_runningTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to IRunningTask handler: " + handler);
}
bridgeHandlerManager.RunningTaskHandler = ClrHandlerHelper.CreateHandler(_runningTaskSubscriber);
// subscribe to Completed Task
foreach (var handler in _completedTaskHandlers)
{
_completedTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to ICompletedTask handler: " + handler);
}
bridgeHandlerManager.CompletedTaskHandler = ClrHandlerHelper.CreateHandler(_completedTaskSubscriber);
// subscribe to Suspended Task
foreach (var handler in _suspendedTaskHandlers)
{
_suspendedTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to ISuspendedTask handler: " + handler);
}
bridgeHandlerManager.SuspendedTaskHandler = ClrHandlerHelper.CreateHandler(_suspendedTaskSubscriber);
// subscribe to Failed Evaluator
foreach (var handler in _failedEvaluatorHandlers)
{
_failedEvaluatorSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to IFailedEvaluator handler: " + handler);
}
bridgeHandlerManager.FailedEvaluatorHandler = ClrHandlerHelper.CreateHandler(_failedEvaluatorSubscriber);
// subscribe to Completed Evaluator
foreach (var handler in _completedEvaluatorHandlers)
{
_completedEvaluatorSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to ICompletedEvaluator handler: " + handler);
}
bridgeHandlerManager.CompletedEvaluatorHandler = ClrHandlerHelper.CreateHandler(_completedEvaluatorSubscriber);
// subscribe to Closed Context
foreach (var handler in _closedContextHandlers)
{
_closedContextSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to IClosedContext handler: " + handler);
}
bridgeHandlerManager.ClosedContextHandler = ClrHandlerHelper.CreateHandler(_closedContextSubscriber);
// subscribe to Failed Context
foreach (var handler in _failedContextHandlers)
{
_failedContextSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to IFailedContext handler: " + handler);
}
bridgeHandlerManager.FailedContextHandler = ClrHandlerHelper.CreateHandler(_failedContextSubscriber);
// subscribe to Context Message
foreach (var handler in _contextMessageHandlers)
{
_contextMessageSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to IContextMesage handler: " + handler);
}
bridgeHandlerManager.ContextMessageHandler = ClrHandlerHelper.CreateHandler(_contextMessageSubscriber);
// subscribe to Active Context received during driver restart
foreach (var handler in _driverRestartActiveContextHandlers)
{
_driverRestartActiveContextSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to handler for IActiveContext received during driver restart: " + handler);
}
bridgeHandlerManager.DriverRestartActiveContextHandler = ClrHandlerHelper.CreateHandler(_driverRestartActiveContextSubscriber);
// subscribe to Running Task received during driver restart
foreach (var handler in _driverRestartRunningTaskHandlers)
{
_driverRestartRunningTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to handler for IRunningTask received during driver restart: " + handler);
}
bridgeHandlerManager.DriverRestartRunningTaskHandler = ClrHandlerHelper.CreateHandler(_driverRestartRunningTaskSubscriber);
// subscribe to Restart Completed received during driver restart
foreach (var handler in _driverRestartCompletedHandlers)
{
_driverRestartCompletedSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to handler for IRestartCompleted received during driver restart: " + handler);
}
bridgeHandlerManager.DriverRestartCompletedHandler = ClrHandlerHelper.CreateHandler(_driverRestartCompletedSubscriber);
// subscribe to Failed Evaluator received during driver restart
foreach (var handler in _driverRestartFailedEvaluatorHandlers)
{
_driverRestartFailedEvaluatorSubscriber.Subscribe(handler);
_logger.Log(Level.Verbose, "subscribed to handler for IFailedEvaluator received during driver restart: " + handler);
}
bridgeHandlerManager.DriverRestartFailedEvaluatorHandler = ClrHandlerHelper.CreateHandler(_driverRestartFailedEvaluatorSubscriber);
// subscribe to Http message
_httpServerEventSubscriber.Subscribe(_httpServerHandler);
_logger.Log(Level.Verbose, "subscribed to IHttpMessage handler :" + _httpServerHandler);
bridgeHandlerManager.HttpServerHandler = ClrHandlerHelper.CreateHandler(_httpServerEventSubscriber);
// bind progress provider to provide application progress
// Bind null handler if user does not specify their own implementation of IProgressProvider. This is
// used to get around the overhead of Interop calls since the Java side checks for null handler here.
bridgeHandlerManager.ProgressProvider = _progressProvider is DefaultProgressProvider?
ClrHandlerHelper.CreateNullHandler() : ClrHandlerHelper.CreateHandler(_progressProvider);
return(bridgeHandlerManager);
}
public ulong[] Subscribe()
{
ulong[] handlers = Enumerable.Repeat(Constants.NullHandler, Constants.HandlersNumber).ToArray();
// subscribe to StartTime event for driver restart
_driverRestartSubscriber.Subscribe(_legacyDriverRestartHandler);
_logger.Log(Level.Info, "subscribed to Driver restart handler: " + _legacyDriverRestartHandler);
handlers[Constants.Handlers[Constants.DriverRestartHandler]] = ClrHandlerHelper.CreateHandler(_driverRestartSubscriber);
// subscribe to Allocated Evaluator
foreach (var handler in _allocatedEvaluatorHandlers)
{
_allocatedEvaluatorSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to IAllocatedEvaluator handler: " + handler);
}
handlers[Constants.Handlers[Constants.AllocatedEvaluatorHandler]] = ClrHandlerHelper.CreateHandler(_allocatedEvaluatorSubscriber);
// subscribe to TaskMessage
foreach (var handler in _taskMessageHandlers)
{
_taskMessageSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to ITaskMessage handler: " + handler);
}
handlers[Constants.Handlers[Constants.TaskMessageHandler]] = ClrHandlerHelper.CreateHandler(_taskMessageSubscriber);
// subscribe to Active Context
foreach (var handler in _activeContextHandlers)
{
_activeContextSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to IActiveContext handler: " + handler);
}
handlers[Constants.Handlers[Constants.ActiveContextHandler]] = ClrHandlerHelper.CreateHandler(_activeContextSubscriber);
// subscribe to Failed Task
foreach (var handler in _failedTaskHandlers)
{
_failedTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to IFailedTask handler: " + handler);
}
handlers[Constants.Handlers[Constants.FailedTaskHandler]] = ClrHandlerHelper.CreateHandler(_failedTaskSubscriber);
// subscribe to Running Task
foreach (var handler in _runningTaskHandlers)
{
_runningTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to IRunningask handler: " + handler);
}
handlers[Constants.Handlers[Constants.RunningTaskHandler]] = ClrHandlerHelper.CreateHandler(_runningTaskSubscriber);
// subscribe to Completed Task
foreach (var handler in _completedTaskHandlers)
{
_completedTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to ICompletedTask handler: " + handler);
}
handlers[Constants.Handlers[Constants.CompletedTaskHandler]] = ClrHandlerHelper.CreateHandler(_completedTaskSubscriber);
// subscribe to Suspended Task
foreach (var handler in _suspendedTaskHandlers)
{
_suspendedTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to ISuspendedTask handler: " + handler);
}
handlers[Constants.Handlers[Constants.SuspendedTaskHandler]] = ClrHandlerHelper.CreateHandler(_suspendedTaskSubscriber);
// subscribe to Failed Evaluator
foreach (var handler in _failedEvaluatorHandlers)
{
_failedEvaluatorSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to IFailedEvaluator handler: " + handler);
}
handlers[Constants.Handlers[Constants.FailedEvaluatorHandler]] = ClrHandlerHelper.CreateHandler(_failedEvaluatorSubscriber);
// subscribe to Completed Evaluator
foreach (var handler in _completedEvaluatorHandlers)
{
_completedEvaluatorSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to ICompletedEvaluator handler: " + handler);
}
handlers[Constants.Handlers[Constants.CompletedEvaluatorHandler]] = ClrHandlerHelper.CreateHandler(_completedEvaluatorSubscriber);
// subscribe to Closed Context
foreach (var handler in _closedContextHandlers)
{
_closedContextSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to IClosedContext handler: " + handler);
}
handlers[Constants.Handlers[Constants.ClosedContextHandler]] = ClrHandlerHelper.CreateHandler(_closedContextSubscriber);
// subscribe to Failed Context
foreach (var handler in _failedContextHandlers)
{
_failedContextSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to IFailedContext handler: " + handler);
}
handlers[Constants.Handlers[Constants.FailedContextHandler]] = ClrHandlerHelper.CreateHandler(_failedContextSubscriber);
// subscribe to Context Message
foreach (var handler in _contextMessageHandlers)
{
_contextMessageSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to IContextMesage handler: " + handler);
}
handlers[Constants.Handlers[Constants.ContextMessageHandler]] = ClrHandlerHelper.CreateHandler(_contextMessageSubscriber);
// subscribe to Active Context received during driver restart
foreach (var handler in _driverRestartActiveContextHandlers)
{
_driverRestartActiveContextSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to handler for IActiveContext received during driver restart: " + handler);
}
handlers[Constants.Handlers[Constants.DriverRestartActiveContextHandler]] = ClrHandlerHelper.CreateHandler(_driverRestartActiveContextSubscriber);
// subscribe to Running Task received during driver restart
foreach (var handler in _driverRestartRunningTaskHandlers)
{
_driverRestartRunningTaskSubscriber.Subscribe(handler);
_logger.Log(Level.Info, "subscribed to handler for IRunningTask received during driver restart: " + handler);
}
handlers[Constants.Handlers[Constants.DriverRestartRunningTaskHandler]] = ClrHandlerHelper.CreateHandler(_driverRestartRunningTaskSubscriber);
// subscribe to Http message
_httpServerEventSubscriber.Subscribe(_httpServerHandler);
_logger.Log(Level.Info, "subscribed to IHttpMessage handler :" + _httpServerHandler);
handlers[Constants.Handlers[Constants.HttpServerHandler]] = ClrHandlerHelper.CreateHandler(_httpServerEventSubscriber);
return(handlers);
}