コード例 #1
0
        private IJobSubmissionResult Run()
        {
            var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
                                           .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloAllocatedEvaluatorHandler> .Class)
                                           .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <AnotherHelloAllocatedEvaluatorHandler> .Class)
                                           .Set(DriverConfiguration.OnContextActive, GenericType <HelloActiveContextHandler> .Class)
                                           .Set(DriverConfiguration.OnTaskMessage, GenericType <HelloTaskMessageHandler> .Class)
                                           .Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloFailedEvaluatorHandler> .Class)
                                           .Set(DriverConfiguration.OnTaskFailed, GenericType <HelloFailedTaskHandler> .Class)
                                           .Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRunningTaskHandler> .Class)
                                           .Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloTaskCompletedHandler> .Class)
                                           .Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriverStartHandler> .Class)
                                           .Set(DriverConfiguration.OnHttpEvent, GenericType <HelloHttpHandler> .Class)
                                           .Set(DriverConfiguration.OnEvaluatorCompleted, GenericType <HelloCompletedEvaluatorHandler> .Class)
                                           .Set(DriverConfiguration.CustomTraceListeners, GenericType <DefaultCustomTraceListener> .Class)
                                           .Set(DriverConfiguration.CustomTraceLevel, Level.Info.ToString())
                                           .Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartHandler> .Class)
                                           .Set(DriverConfiguration.DriverReconnectionConfigurationProvider,
                                                GenericType <LocalHttpDriverReconnConfigProvider> .Class)
                                           .Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloDriverRestartActiveContextHandler> .Class)
                                           .Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloDriverRestartRunningTaskHandler> .Class)
                                           .Build();

            var helloJobSubmission = _jobRequestBuilder
                                     .AddDriverConfiguration(helloDriverConfiguration)
                                     .AddGlobalAssemblyForType(typeof(HelloDriverStartHandler))
                                     .SetJobIdentifier("HelloDriver")
                                     .Build();

            IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobSubmission);

            return(jobSubmissionResult);
        }
コード例 #2
0
ファイル: TestBridgeClient.cs プロジェクト: wwjiang007/reef
        private async Task RunClrBridgeClient(bool runOnYarn, string testRuntimeFolder)
        {
            string[]             a = { runOnYarn ? "yarn" : "local", testRuntimeFolder };
            IJobSubmissionResult driverHttpEndpoint = AllHandlers.Run(a);

            var driverUrl = driverHttpEndpoint.DriverUrl;

            int retryCount = 1;

            while (string.IsNullOrEmpty(driverUrl) && retryCount < 10)
            {
                driverUrl = driverHttpEndpoint.DriverUrl;
                retryCount++;
            }

            if (driverUrl != null)
            {
                var uri       = driverUrl + "NRT/status?a=1&b=2";
                var strStatus = driverHttpEndpoint.GetUrlResult(uri);
                Assert.NotNull(strStatus);
                Assert.True(strStatus.Equals("Byte array returned from HelloHttpHandler in CLR!!!\r\n"));

                var uri1       = driverUrl + "CMD/Stop?c=1";
                var strStatus1 = driverHttpEndpoint.GetUrlResult(uri1);

                Assert.NotNull(strStatus1);
                Assert.True(strStatus1.Equals("Stopped!!!\r\n"));

                await((JobSubmissionResult)driverHttpEndpoint).TryUntilNoConnection(uri);
            }
        }
コード例 #3
0
        /// <summary>
        /// Runs HelloREEF using the IREEFClient passed into the constructor.
        /// </summary>
        private void Run()
        {
            // The driver configuration contains all the needed bindings.
            var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
                                           .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloDriver> .Class)
                                           .Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriver> .Class)
                                           .Set(DriverConfiguration.CustomTraceLevel, Level.Verbose.ToString())
                                           .Build();

            string applicationId = GetApplicationId();

            // The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
            var helloJobRequest = _reefClient.NewJobRequestBuilder()
                                  .AddDriverConfiguration(helloDriverConfiguration)
                                  .AddGlobalAssemblyForType(typeof(HelloDriver))
                                  .AddGlobalAssembliesInDirectoryOfExecutingAssembly()
                                  .SetJobIdentifier(applicationId)
                                  .SetJavaLogLevel(JavaLoggingSetting.Verbose)
                                  .Build();

            IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobRequest);

            // Wait for the Driver to complete.
            jobSubmissionResult?.WaitForDriverToFinish();
        }
コード例 #4
0
ファイル: LocalTestRunner.cs プロジェクト: wwjiang007/reef
        public ITestResult RunTest(JobRequestBuilder jobRequestBuilder)
        {
            // Setup the assert file.
            var assertFileName = Path.GetTempPath() + "/reef-test-" + DateTime.Now.ToString("yyyyMMddHHmmssfff") + ".json";

            jobRequestBuilder.AddDriverConfiguration(FileWritingAssertConfiguration.ConfigurationModule
                                                     .Set(FileWritingAssertConfiguration.FilePath, assertFileName)
                                                     .Build());
            var jobRequest = jobRequestBuilder.Build();

            LOG.Log(Level.Info, "Submitting job `{0}` for execution. Assert log in `{1}`",
                    jobRequest.JobIdentifier,
                    assertFileName);
            IJobSubmissionResult jobStatus = _client.SubmitAndGetJobStatus(jobRequest);

            if (jobStatus == null)
            {
                return(TestResult.Fail(
                           "JobStatus returned by the Client was null. This points to an environment setup problem."));
            }

            LOG.Log(Level.Verbose, "Waiting for job `{0}` to complete.", jobRequest.JobIdentifier);
            jobStatus.WaitForDriverToFinish();
            LOG.Log(Level.Verbose, "Job `{0}` completed.", jobRequest.JobIdentifier);

            return(ReadTestResult(assertFileName));
        }
コード例 #5
0
ファイル: AllHandlers.cs プロジェクト: TaeHunKim/reef
        /// <summary>
        /// </summary>
        /// <param name="args"></param>
        /// <remarks>
        /// args[0] specify either running local or YARN. Default is local
        /// args[1] specify running folder. Default is REEF_LOCAL_RUNTIME
        /// </remarks>
        public static IJobSubmissionResult Run(string[] args)
        {
            string runOnYarn     = args.Length > 0 ? args[0] : Local;
            string runtimeFolder = args.Length > 1 ? args[1] : "REEF_LOCAL_RUNTIME";
            IJobSubmissionResult jobSubmissionResult = TangFactory.GetTang().NewInjector(GetRuntimeConfiguration(runOnYarn, runtimeFolder)).GetInstance <AllHandlers>().Run();

            return(jobSubmissionResult);
        }
コード例 #6
0
        /// <summary>
        /// Sample code to pull job final status until the Job is done
        /// </summary>
        /// <param name="jobSubmitionResult"></param>
        /// <returns></returns>
        private FinalState PullFinalJobStatus(IJobSubmissionResult jobSubmitionResult)
        {
            int n     = 0;
            var state = jobSubmitionResult.FinalState;

            while (state.Equals(FinalState.UNDEFINED) && n++ < ReTryCounts)
            {
                Thread.Sleep(SleepTime);
                state = jobSubmitionResult.FinalState;
            }
            return(state);
        }
コード例 #7
0
ファイル: TestBridgeClient.cs プロジェクト: rongxunw/reef
        private async Task RunClrBridgeClient(bool runOnYarn, string testRuntimeFolder)
        {
            string[]             a = { runOnYarn ? "yarn" : "local", testRuntimeFolder };
            IJobSubmissionResult driverHttpEndpoint = AllHandlers.Run(a);

            var uri       = driverHttpEndpoint.DriverUrl + "NRT/status?a=1&b=2";
            var strStatus = driverHttpEndpoint.GetUrlResult(uri);

            Assert.NotNull(strStatus);
            Assert.True(strStatus.Equals("Byte array returned from HelloHttpHandler in CLR!!!\r\n"));

            await((JobSubmissionResult)driverHttpEndpoint).TryUntilNoConnection(uri);
        }
コード例 #8
0
ファイル: TestBridgeClient.cs プロジェクト: TaeHunKim/reef
        private async void RunClrBridgeClient(bool runOnYarn, string testRuntimeFolder)
        {
            string[]             a = new[] { runOnYarn ? "yarn" : "local", testRuntimeFolder };
            IJobSubmissionResult driverHttpEndpoint = AllHandlers.Run(a);

            var uri       = driverHttpEndpoint.DriverUrl + "NRT/status?a=1&b=2";
            var strStatus = driverHttpEndpoint.GetUrlResult(uri);

            Assert.IsTrue(strStatus.Equals("Byte array returned from HelloHttpHandler in CLR!!!\r\n"));

            await((JobSubmissionResult)driverHttpEndpoint).TryUntilNoConnection(uri);

            ValidateSuccessForLocalRuntime(2, testFolder: testRuntimeFolder);

            CleanUp(testRuntimeFolder);
        }
コード例 #9
0
ファイル: AllHandlers.cs プロジェクト: TaeHunKim/reef
        private IJobSubmissionResult Run()
        {
            var helloDriverConfiguration = DriverConfiguration.ConfigurationModule
                                           .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <HelloAllocatedEvaluatorHandler> .Class)
                                           .Set(DriverConfiguration.OnEvaluatorAllocated, GenericType <AnotherHelloAllocatedEvaluatorHandler> .Class)
                                           .Set(DriverConfiguration.OnContextActive, GenericType <HelloActiveContextHandler> .Class)
                                           .Set(DriverConfiguration.OnTaskMessage, GenericType <HelloTaskMessageHandler> .Class)
                                           .Set(DriverConfiguration.OnEvaluatorFailed, GenericType <HelloFailedEvaluatorHandler> .Class)
                                           .Set(DriverConfiguration.OnTaskFailed, GenericType <HelloFailedTaskHandler> .Class)
                                           .Set(DriverConfiguration.OnTaskRunning, GenericType <HelloRunningTaskHandler> .Class)
                                           .Set(DriverConfiguration.OnTaskCompleted, GenericType <HelloTaskCompletedHandler> .Class)
                                           .Set(DriverConfiguration.OnDriverStarted, GenericType <HelloDriverStartHandler> .Class)
                                           .Set(DriverConfiguration.OnHttpEvent, GenericType <HelloHttpHandler> .Class)
                                           .Set(DriverConfiguration.OnEvaluatorCompleted, GenericType <HelloCompletedEvaluatorHandler> .Class)
                                           .Set(DriverConfiguration.CustomTraceListeners, GenericType <DefaultCustomTraceListener> .Class)
                                           .Set(DriverConfiguration.CustomTraceLevel, Level.Info.ToString())
                                           .Set(DriverConfiguration.OnDriverRestarted, GenericType <HelloRestartHandler> .Class)
                                           .Set(DriverConfiguration.OnDriverReconnect, GenericType <DefaultLocalHttpDriverConnection> .Class)
                                           .Set(DriverConfiguration.OnDriverRestartContextActive, GenericType <HelloDriverRestartActiveContextHandler> .Class)
                                           .Set(DriverConfiguration.OnDriverRestartTaskRunning, GenericType <HelloDriverRestartRunningTaskHandler> .Class)
                                           .Build();

            var driverConfig = TangFactory.GetTang().NewConfigurationBuilder(helloDriverConfiguration)
                               .BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(HelloTask).Assembly.GetName().Name)
                               .BindSetEntry <DriverBridgeConfigurationOptions.SetOfAssemblies, string>(typeof(NameClient).Assembly.GetName().Name)
                               .Build();

            var helloJobSubmission = _jobSubmissionBuilderFactory.GetJobSubmissionBuilder()
                                     .AddDriverConfiguration(driverConfig)
                                     .AddGlobalAssemblyForType(typeof(HelloDriverStartHandler))
                                     .SetJobIdentifier("HelloDriver")
                                     .Build();

            IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobSubmission);

            return(jobSubmissionResult);
        }
コード例 #10
0
ファイル: REEFIMRUClient.cs プロジェクト: shulmanb/reef
        /// <summary>
        /// Submits the job to reefClient
        /// </summary>
        /// <typeparam name="TMapInput">The type of the side information provided to the Map function</typeparam>
        /// <typeparam name="TMapOutput">The return type of the Map function</typeparam>
        /// <typeparam name="TResult">The return type of the computation.</typeparam>
        /// <typeparam name="TPartitionType">Type of data partition (Generic type in IInputPartition)</typeparam>
        /// <param name="jobDefinition">IMRU job definition given by the user</param>
        /// <returns>Null as results will be later written to some directory</returns>
        IEnumerable <TResult> IIMRUClient.Submit <TMapInput, TMapOutput, TResult, TPartitionType>(IMRUJobDefinition jobDefinition)
        {
            string         driverId            = string.Format("IMRU-{0}-Driver", jobDefinition.JobName);
            IConfiguration overallPerMapConfig = null;

            try
            {
                overallPerMapConfig = Configurations.Merge(jobDefinition.PerMapConfigGeneratorConfig.ToArray());
            }
            catch (Exception e)
            {
                Exceptions.Throw(e, "Issues in merging PerMapCOnfigGenerator configurations", Logger);
            }

            // The driver configuration contains all the needed bindings.
            var imruDriverConfiguration = TangFactory.GetTang().NewConfigurationBuilder(new[]
            {
                DriverConfiguration.ConfigurationModule
                .Set(DriverConfiguration.OnEvaluatorAllocated,
                     GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
                .Set(DriverConfiguration.OnDriverStarted,
                     GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
                .Set(DriverConfiguration.OnContextActive,
                     GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
                .Set(DriverConfiguration.OnTaskCompleted,
                     GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
                .Set(DriverConfiguration.OnEvaluatorFailed,
                     GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
                .Set(DriverConfiguration.OnContextFailed,
                     GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
                .Set(DriverConfiguration.OnTaskFailed,
                     GenericType <IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType> > .Class)
                .Set(DriverConfiguration.CustomTraceLevel, TraceLevel.Info.ToString())
                .Build(),
                TangFactory.GetTang().NewConfigurationBuilder()
                .BindStringNamedParam <GroupCommConfigurationOptions.DriverId>(driverId)
                .BindStringNamedParam <GroupCommConfigurationOptions.MasterTaskId>(IMRUConstants.UpdateTaskName)
                .BindStringNamedParam <GroupCommConfigurationOptions.GroupName>(IMRUConstants.CommunicationGroupName)
                .BindIntNamedParam <GroupCommConfigurationOptions.FanOut>(
                    IMRUConstants.TreeFanout.ToString(CultureInfo.InvariantCulture)
                    .ToString(CultureInfo.InvariantCulture))
                .BindIntNamedParam <GroupCommConfigurationOptions.NumberOfTasks>(
                    (jobDefinition.NumberOfMappers + 1).ToString(CultureInfo.InvariantCulture))
                .BindImplementation(GenericType <IGroupCommDriver> .Class, GenericType <GroupCommDriver> .Class)
                .Build(),
                jobDefinition.PartitionedDatasetConfiguration,
                overallPerMapConfig
            })
                                          .BindNamedParameter(typeof(SerializedMapConfiguration),
                                                              _configurationSerializer.ToString(jobDefinition.MapFunctionConfiguration))
                                          .BindNamedParameter(typeof(SerializedUpdateConfiguration),
                                                              _configurationSerializer.ToString(jobDefinition.UpdateFunctionConfiguration))
                                          .BindNamedParameter(typeof(SerializedMapInputCodecConfiguration),
                                                              _configurationSerializer.ToString(jobDefinition.MapInputCodecConfiguration))
                                          .BindNamedParameter(typeof(SerializedMapInputPipelineDataConverterConfiguration),
                                                              _configurationSerializer.ToString(jobDefinition.MapInputPipelineDataConverterConfiguration))
                                          .BindNamedParameter(typeof(SerializedUpdateFunctionCodecsConfiguration),
                                                              _configurationSerializer.ToString(jobDefinition.UpdateFunctionCodecsConfiguration))
                                          .BindNamedParameter(typeof(SerializedMapOutputPipelineDataConverterConfiguration),
                                                              _configurationSerializer.ToString(jobDefinition.MapOutputPipelineDataConverterConfiguration))
                                          .BindNamedParameter(typeof(SerializedReduceConfiguration),
                                                              _configurationSerializer.ToString(jobDefinition.ReduceFunctionConfiguration))
                                          .BindNamedParameter(typeof(SerializedResultHandlerConfiguration),
                                                              _configurationSerializer.ToString(jobDefinition.ResultHandlerConfiguration))
                                          .BindNamedParameter(typeof(MemoryPerMapper),
                                                              jobDefinition.MapperMemory.ToString(CultureInfo.InvariantCulture))
                                          .BindNamedParameter(typeof(MemoryForUpdateTask),
                                                              jobDefinition.UpdateTaskMemory.ToString(CultureInfo.InvariantCulture))
                                          .BindNamedParameter(typeof(CoresPerMapper),
                                                              jobDefinition.MapTaskCores.ToString(CultureInfo.InvariantCulture))
                                          .BindNamedParameter(typeof(CoresForUpdateTask),
                                                              jobDefinition.UpdateTaskCores.ToString(CultureInfo.InvariantCulture))
                                          .BindNamedParameter(typeof(InvokeGC),
                                                              jobDefinition.InvokeGarbageCollectorAfterIteration.ToString(CultureInfo.InvariantCulture))
                                          .Build();

            // The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
            var imruJobSubmission = _jobRequestBuilder
                                    .AddDriverConfiguration(imruDriverConfiguration)
                                    .AddGlobalAssemblyForType(typeof(IMRUDriver <TMapInput, TMapOutput, TResult, TPartitionType>))
                                    .SetJobIdentifier(jobDefinition.JobName)
                                    .SetDriverMemory(5000)
                                    .Build();

            _jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(imruJobSubmission);

            return(null);
        }