// [Fact] public void GetPigJob() { using (var context = UndoContext.Current) { context.Start(); var username = TestUtils.UserName; var password = TestUtils.Password; var clustername = TestUtils.ClusterName; var credentials = new BasicAuthenticationCloudCredentials { Username = username, Password = password }; var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials); var parameters = new PigJobSubmissionParameters() { UserName = username, Query = "records = LOAD '/example/pig/sahara-paleo-fauna.txt' AS (DateBP:int, Loc:chararray, Coordinates:chararray, Samples:chararray, Country:chararray, Laboratory:chararray);" + "filtered_records = FILTER records by Country == 'Egypt' OR Country == 'Morocco';" + "grouped_records = GROUP filtered_records BY Country;" + "DUMP grouped_records;" }; var jobid = client.JobManagement.SubmitPigJob(parameters).JobSubmissionJsonResponse.Id; var response = client.JobManagement.GetJob(jobid); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); } }
public virtual JobSubmissionResponse SubmitPigJob(AzureHDInsightPigJobDefinition pigJobDef) { var pigJobParams = new PigJobSubmissionParameters { Arguments = pigJobDef.Arguments, Files = pigJobDef.Files, StatusDir = pigJobDef.StatusFolder, File = pigJobDef.File, Query = pigJobDef.Query }; return(HdInsightJobManagementClient.JobManagement.SubmitPigJob(pigJobParams)); }
public virtual JobSubmissionResponse SubmitPigJob(AzureHDInsightPigJobDefinition pigJobDef) { var pigJobParams = new PigJobSubmissionParameters { Arguments = ConvertListToString(pigJobDef.Arguments, "arg"), Files = ConvertListToString(pigJobDef.Files, "file"), StatusDir = pigJobDef.StatusFolder, File = pigJobDef.File, Query = pigJobDef.Query, UserName = HdInsightJobManagementClient.Credentials.Username }; return(HdInsightJobManagementClient.JobManagement.SubmitPigJob(pigJobParams)); }
public static void SubmitPigJob() { const string ExistingClusterName = "sparkdheetest"; const string ExistingClusterUri = ExistingClusterName + ".azurehdinsight.net"; const string ExistingClusterUsername = "******"; const string ExistingClusterPassword = "******"; var clusterCredentials = new BasicAuthenticationCloudCredentials { Username = ExistingClusterUsername, Password = ExistingClusterPassword }; HDInsightJobManagementClient _hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials); ////SubmitPigJob(); var parameters = new PigJobSubmissionParameters { //A = LOAD 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/information.txt' using PigStorage (‘\t’) as (FName: chararray, LName: chararray, MobileNo: chararray, City: chararray, Profession: chararray); // B = FOREACH A generate FName,MobileNo,Profession; //DUMP B; //wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/sample.pig //Query = @"PigSampleIn = LOAD 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/input.txt' USIG PigStorage(',') AS (ProfileID:chararray, SessionStart:chararray, Duration:int, SrcIPAddress:chararray, GameType:chararray); // GroupProfile = Group PigSampleIn all; // PigSampleOut = Foreach GroupProfile Generate PigSampleIn.ProfileID, SUM(PigSampleIn.Duration); // Store PigSampleOut into 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/output.txt' USING PigStorage (',');" // File = }; Console.WriteLine("Submitting the Pig job to the cluster..."); var response = _hdiJobManagementClient.JobManagement.SubmitPigJob(parameters); Console.WriteLine("Validating that the response is as expected..."); Console.WriteLine("Response status code is " + response.StatusCode); Console.WriteLine("Validating the response object..."); Console.WriteLine("JobId is " + response.JobSubmissionJsonResponse.Id); var jobs = _hdiJobManagementClient.JobManagement.ListJobs(); foreach (var job in jobs) { //var deails = _hdiJobManagementClient.JobManagement.GetJobOutput(job.Id) } Console.WriteLine("Press ENTER to continue ..."); Console.ReadLine(); }
public void SubmitPigJobAndValidateOutput(PigJobSubmissionParameters parameters, bool runAyncAPI = false, bool isWindowsCluster = false) { using (var context = UndoContext.Current) { context.Start(); var client = TestUtils.GetHDInsightJobManagementClient(isWindowsCluster); var response = runAyncAPI ? client.JobManagement.SubmitPigJobAsync(parameters).Result : client.JobManagement.SubmitPigJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); var jobId = response.JobSubmissionJsonResponse.Id; Assert.Contains("job_", jobId, StringComparison.InvariantCulture); WaitForJobCompletion(client, jobId, TestUtils.JobPollInterval, TestUtils.JobWaitInterval); var jobStatus = client.JobManagement.GetJob(jobId); var storageAccess = GetStorageAccessObject(isWindowsCluster); if (jobStatus.JobDetail.ExitValue == 0) { // Retrieve Job Output if (HttpMockServer.Mode == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobOutput(jobId, storageAccess); string textOutput = Convert(output); Assert.True(textOutput.Contains("(DEBUG,")); } } else { if (HttpMockServer.Mode == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess); string errorTextOutput = Convert(output); Assert.NotNull(errorTextOutput); } Assert.True(false); } } }
private void SubmitPigJobAndValidateOutput(PigJobSubmissionParameters parameters, bool runAyncAPI = false) { using (var context = MockContext.Start(this.GetType())) using (var client = this.CommonData.GetHDInsightJobClient(context)) { var response = runAyncAPI ? client.Job.SubmitPigJobAsync(parameters).Result : client.Job.SubmitPigJob(parameters); Assert.NotNull(response); var jobId = response.Id; Assert.Contains("job_", jobId, StringComparison.InvariantCulture); var jobStatus = client.Job.WaitForJobCompletion(jobId, this.CommonData.JobWaitInterval, this.CommonData.JobPollInterval); var storageAccess = GetStorageAccessObject(); if (jobStatus.ExitValue == 0) { // Retrieve Job Output if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record) { var output = client.Job.GetJobOutput(jobId, storageAccess); string textOutput = Convert(output); Assert.Contains("(DEBUG,", textOutput); } } else { if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record) { var output = client.Job.GetJobErrorLogs(jobId, storageAccess); string errorTextOutput = Convert(output); Assert.NotNull(errorTextOutput); } Assert.True(false); } } }
public static void SubmitPigJob(string filename) { const string ExistingClusterName = "orionml"; const string ExistingClusterUri = ExistingClusterName + ".azurehdinsight.net"; const string ExistingClusterUsername = "******"; const string ExistingClusterPassword = "******"; var clusterCredentials = new BasicAuthenticationCloudCredentials { Username = ExistingClusterUsername, Password = ExistingClusterPassword }; HDInsightJobManagementClient _hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials); var parameters = new PigJobSubmissionParameters { File = "wasbs:///user/root/" + filename + ".pig" }; Console.WriteLine("Submitting the Pig job with file name [" + filename + "] to the cluster..."); var response = _hdiJobManagementClient.JobManagement.SubmitPigJob(parameters); Console.WriteLine("JobId is " + response.JobSubmissionJsonResponse.Id); }
/// <summary> /// Submits a Pig job to an HDInsight cluster. /// </summary> /// <param name='parameters'> /// Required. Pig job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public async Task <JobSubmissionResponse> SubmitPigJobAsync(PigJobSubmissionParameters parameters) { return(await SubmitPigJobAsync(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() }, CancellationToken.None)); }
/// <summary> /// Submits a Pig job to an HDInsight cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Pig job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static async Task <JobSubmissionJsonResponse> SubmitPigJobAsync(this IJobOperations operations, PigJobSubmissionParameters parameters, CancellationToken cancellationToken = default(CancellationToken)) { using (var _result = await operations.SubmitPigJobWithHttpMessagesAsync(parameters, null, cancellationToken).ConfigureAwait(false)) { return(_result.Body); } }
/// <summary> /// Submits a Pig job to an HDInsight cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Pig job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static JobSubmissionJsonResponse SubmitPigJob(this IJobOperations operations, PigJobSubmissionParameters parameters) { return(operations.SubmitPigJobAsync(parameters).GetAwaiter().GetResult()); }
/// <summary> /// Submits a Pig job to an HDInsight cluster. /// </summary> /// <param name='parameters'> /// Required. Pig job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public async Task <AzureOperationResponse <JobSubmissionJsonResponse> > SubmitPigJobWithHttpMessagesAsync(PigJobSubmissionParameters parameters, Dictionary <string, List <string> > customHeaders = null, CancellationToken cancellationToken = default(CancellationToken)) { using (var requestContents = new MemoryStream(Encoding.UTF8.GetBytes(parameters.GetJobPostRequestContent()))) { return(await SubmitPigJobWithHttpMessagesAsync(requestContents, customHeaders, cancellationToken).ConfigureAwait(false)); } }
/// <summary> /// Submits an Hive job to an HDINSIGHT cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Pig job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static Task <JobSubmissionResponse> SubmitPigJobAsync(this IJobOperations operations, PigJobSubmissionParameters parameters) { return(operations.SubmitPigJobAsync(parameters, CancellationToken.None)); }
/// <summary> /// Submits an Hive job to an HDINSIGHT cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Pig job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static JobSubmissionResponse SubmitPigJob(this IJobOperations operations, PigJobSubmissionParameters parameters) { return(Task.Factory.StartNew((object s) => { return ((IJobOperations)s).SubmitPigJobAsync(parameters); } , operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult()); }
public void SubmitPigJob() { using (var context = UndoContext.Current) { context.Start(); var username = TestUtils.UserName; var password = TestUtils.Password; var clustername = TestUtils.ClusterName; var credentials = new BasicAuthenticationCloudCredentials() { Username = username, Password = password }; var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials); var parameters = new PigJobSubmissionParameters() { Query = "LOGS = LOAD 'wasb:///example/data/sample.log';" + "LEVELS = foreach LOGS generate REGEX_EXTRACT($0, '(TRACE|DEBUG|INFO|WARN|ERROR|FATAL)', 1) as LOGLEVEL;" + "FILTEREDLEVELS = FILTER LEVELS by LOGLEVEL is not null;" + "GROUPEDLEVELS = GROUP FILTEREDLEVELS by LOGLEVEL;" + "FREQUENCIES = foreach GROUPEDLEVELS generate group as LOGLEVEL, COUNT(FILTEREDLEVELS.LOGLEVEL) as COUNT;" + "RESULT = order FREQUENCIES by COUNT desc;" + "DUMP RESULT;" }; var response = client.JobManagement.SubmitPigJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); var jobId = response.JobSubmissionJsonResponse.Id; Assert.Contains("job_", jobId, StringComparison.InvariantCulture); var jobStatus = GetJobFinalStatus(client, jobId); var storageAccess = GetStorageAccessObject(); if (jobStatus.JobDetail.ExitValue == 0) { // Retrieve Job Output if (HttpMockServer.Mode == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobOutput(jobId, storageAccess); string textOutput = Convert(output); Assert.True(textOutput.Contains("(DEBUG,")); } } else { if (HttpMockServer.Mode == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess); string errorTextOutput = Convert(output); Assert.NotNull(errorTextOutput); } Assert.True(false); } } }
/// <summary> /// Submits a Pig job to an HDInsight cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Pig job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static JobSubmissionResponse SubmitPigJob(this IJobOperations operations, PigJobSubmissionParameters parameters) { return(operations.SubmitPigJob(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() })); }