public static void SubmitJobToCluster(HDInsightJobManagementClient hdiJobManagementClient, HiveJobSubmissionParameters job) { System.Console.WriteLine("Submitting the Hive job to the cluster..."); var jobResponse = hdiJobManagementClient.JobManagement.SubmitHiveJob(job); var jobId = jobResponse.JobSubmissionJsonResponse.Id; System.Console.WriteLine("Response status code is " + jobResponse.StatusCode); System.Console.WriteLine("JobId is " + jobId); System.Console.WriteLine("Waiting for the job completion ..."); // Wait for job completion var jobDetail = hdiJobManagementClient.JobManagement.GetJob(jobId).JobDetail; while (!jobDetail.Status.JobComplete) { Thread.Sleep(1000); jobDetail = hdiJobManagementClient.JobManagement.GetJob(jobId).JobDetail; } // Get job output var storageAccess = new AzureStorageAccess(storageAccount, storageKey, storageContainer); var output = (jobDetail.ExitValue == 0) ? hdiJobManagementClient.JobManagement.GetJobOutput(jobId, storageAccess) // fetch stdout output in case of success : hdiJobManagementClient.JobManagement.GetJobErrorLogs(jobId, storageAccess); // fetch stderr output in case of failure System.Console.WriteLine("Job output is: "); using (var reader = new StreamReader(output, Encoding.UTF8)) { string value = reader.ReadToEnd(); System.Console.WriteLine(value); } }
public InsightPhotoController(IHostingEnvironment hostingEnv) { _env = hostingEnv; var clusterCredentials = new BasicAuthenticationCloudCredentials { Username = ExistingClusterUsername, Password = ExistingClusterPassword }; _jobClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials); }
private JobGetResponse GetJobFinalStatus(HDInsightJobManagementClient client, string jobId) { var jobStatus = client.JobManagement.GetJob(jobId); while (!jobStatus.JobDetail.Status.JobComplete) { jobStatus = client.JobManagement.GetJob(jobId); // Optional to sleep here. Sleep for 1 sec instead of keep pooling. Thread.Sleep(1000); } return(jobStatus); }
static void Main(string[] args) { Console.WriteLine("The application is running ..."); var clusterCredentials = new BasicAuthenticationCloudCredentials { Username = ExistingClusterUsername, Password = ExistingClusterPassword }; _hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials); SubmitMRJob(); Console.WriteLine("Press ENTER to continue ..."); Console.ReadLine(); }
public void CheckValidJobUserName() { var credentials = new BasicAuthenticationCredentials { UserName = "******", Password = "" }; var client = new HDInsightJobManagementClient("TestCluster", credentials); Assert.Equal(credentials.UserName.ToLower(CultureInfo.CurrentCulture), client.Username); client = new HDInsightJobManagementClient("TestCluster", credentials, new HttpClient()); Assert.Equal(credentials.UserName.ToLower(CultureInfo.CurrentCulture), client.Username); }
public RecalculationJob(StorageConfiguration storageConfiguration, HDInsightConfiguration configuration) { _storageConfiguration = storageConfiguration; _configuration = configuration; _storageAccess = new AzureStorageAccess( _configuration.DefaultStorageAccountName, _configuration.DefaultStorageAccountKey, _configuration.DefaultStorageContainerName); var clusterCredentials = new BasicAuthenticationCloudCredentials { Username = _configuration.ExistingClusterUsername, Password = _configuration.ExistingClusterPassword }; _hdiJobManagementClient = new HDInsightJobManagementClient(_configuration.ExistingClusterUri, clusterCredentials); }
static void Main(string[] args) { var key = new SecureString(); foreach (char c in secretKey) { key.AppendChar(c); } var tokenCreds = GetTokenCloudCredentials(tenantId, appId, key); var resourceManagementClient = new ResourceManagementClient(new TokenCloudCredentials(subId.ToString(), tokenCreds.Token)); resourceManagementClient.Providers.Register("Microsoft.HDInsight"); var hdiManagementClient = new HDInsightManagementClient(new TokenCredentials(tokenCreds.Token)); hdiManagementClient.SubscriptionId = subId.ToString(); var clusterName = GetClusterName(hdiManagementClient); Console.ReadLine(); var hdiJobManagementClient = new HDInsightJobManagementClient(clusterName + ".azurehdinsight.net", new BasicAuthenticationCloudCredentials { Username = user, Password = password }); Dictionary <string, string> defines = new Dictionary <string, string> { { "hive.execution.engine", "tez" }, { "hive.exec.reducers.max", "1" } }; List <string> hadoopArgs = new List <string> { { "argA" }, { "argB" } }; var job = new HiveJobSubmissionParameters { Query = "SHOW TABLES", Defines = defines, Arguments = hadoopArgs }; SubmitJobToCluster(hdiJobManagementClient, job); Console.ReadLine(); }
public static void SubmitPigJob() { const string ExistingClusterName = "sparkdheetest"; const string ExistingClusterUri = ExistingClusterName + ".azurehdinsight.net"; const string ExistingClusterUsername = "******"; const string ExistingClusterPassword = "******"; var clusterCredentials = new BasicAuthenticationCloudCredentials { Username = ExistingClusterUsername, Password = ExistingClusterPassword }; HDInsightJobManagementClient _hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials); ////SubmitPigJob(); var parameters = new PigJobSubmissionParameters { //A = LOAD 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/information.txt' using PigStorage (‘\t’) as (FName: chararray, LName: chararray, MobileNo: chararray, City: chararray, Profession: chararray); // B = FOREACH A generate FName,MobileNo,Profession; //DUMP B; //wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/sample.pig //Query = @"PigSampleIn = LOAD 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/input.txt' USIG PigStorage(',') AS (ProfileID:chararray, SessionStart:chararray, Duration:int, SrcIPAddress:chararray, GameType:chararray); // GroupProfile = Group PigSampleIn all; // PigSampleOut = Foreach GroupProfile Generate PigSampleIn.ProfileID, SUM(PigSampleIn.Duration); // Store PigSampleOut into 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/output.txt' USING PigStorage (',');" // File = }; Console.WriteLine("Submitting the Pig job to the cluster..."); var response = _hdiJobManagementClient.JobManagement.SubmitPigJob(parameters); Console.WriteLine("Validating that the response is as expected..."); Console.WriteLine("Response status code is " + response.StatusCode); Console.WriteLine("Validating the response object..."); Console.WriteLine("JobId is " + response.JobSubmissionJsonResponse.Id); var jobs = _hdiJobManagementClient.JobManagement.ListJobs(); foreach (var job in jobs) { //var deails = _hdiJobManagementClient.JobManagement.GetJobOutput(job.Id) } Console.WriteLine("Press ENTER to continue ..."); Console.ReadLine(); }
public void CheckValidJobUserName() { using (var context = UndoContext.Current) { context.Start(); var credentials = new BasicAuthenticationCloudCredentials { Username = "******", Password = "" }; var client = new HDInsightJobManagementClient("TestCluster", credentials); Assert.Equal(credentials.Username.ToLower(CultureInfo.CurrentCulture), client.UserName); client = new HDInsightJobManagementClient("TestCluster", credentials, new HttpClient()); Assert.Equal(credentials.Username.ToLower(CultureInfo.CurrentCulture), client.UserName); } }
public static void SubmitPigJob(string filename) { const string ExistingClusterName = "orionml"; const string ExistingClusterUri = ExistingClusterName + ".azurehdinsight.net"; const string ExistingClusterUsername = "******"; const string ExistingClusterPassword = "******"; var clusterCredentials = new BasicAuthenticationCloudCredentials { Username = ExistingClusterUsername, Password = ExistingClusterPassword }; HDInsightJobManagementClient _hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials); var parameters = new PigJobSubmissionParameters { File = "wasbs:///user/root/" + filename + ".pig" }; Console.WriteLine("Submitting the Pig job with file name [" + filename + "] to the cluster..."); var response = _hdiJobManagementClient.JobManagement.SubmitPigJob(parameters); Console.WriteLine("JobId is " + response.JobSubmissionJsonResponse.Id); }
internal void WaitForJobCompletion(HDInsightJobManagementClient client, string jobId, TimeSpan pollingInterval, TimeSpan duration, bool cancelJob = false) { var startTime = DateTime.UtcNow; var endTime = DateTime.UtcNow; var jobDetail = client.JobManagement.GetJob(jobId); while (!jobDetail.JobDetail.Status.JobComplete) { if (((endTime = DateTime.UtcNow) - startTime) > duration) { string exceptionMessage = string.Format(CultureInfo.InvariantCulture, "The requested task failed to complete in the allotted time ({0})", duration);; if (cancelJob) { client.JobManagement.KillJob(jobId); exceptionMessage = string.Format(CultureInfo.InvariantCulture, "{0} Killed the Job {1}", exceptionMessage, jobId); } throw new CloudException(exceptionMessage); } jobDetail = client.JobManagement.GetJob(jobId); } }
private JobGetResponse GetJobFinalStatus(HDInsightJobManagementClient client, string jobId) { var jobStatus = client.JobManagement.GetJob(jobId); while (!jobStatus.JobDetail.Status.JobComplete) { jobStatus = client.JobManagement.GetJob(jobId); // Optional to sleep here. Sleep for 1 sec instead of keep pooling. Thread.Sleep(1000); } return jobStatus; }
public static HDInsightJobManagementClient GetHDInsightJobManagementClient(string dnsName, BasicAuthenticationCloudCredentials creds) { var client = new HDInsightJobManagementClient(dnsName, creds); return(AddMockHandler(ref client)); }
internal void WaitForJobCompletion(HDInsightJobManagementClient client, string jobId, TimeSpan pollingInterval, TimeSpan duration, bool cancelJob = false) { var startTime = DateTime.UtcNow; var endTime = DateTime.UtcNow; var jobDetail = client.JobManagement.GetJob(jobId); while (!jobDetail.JobDetail.Status.JobComplete) { if (((endTime = DateTime.UtcNow) - startTime) > duration) { string exceptionMessage = string.Format(CultureInfo.InvariantCulture, "The requested task failed to complete in the allotted time ({0})", duration); ; if (cancelJob) { client.JobManagement.KillJob(jobId); exceptionMessage = string.Format(CultureInfo.InvariantCulture, "{0} Killed the Job {1}", exceptionMessage, jobId); } throw new CloudException(exceptionMessage); } jobDetail = client.JobManagement.GetJob(jobId); } }