public static void SubmitJobToCluster(HDInsightJobManagementClient hdiJobManagementClient, HiveJobSubmissionParameters job)
{
System.Console.WriteLine("Submitting the Hive job to the cluster...");
var jobResponse = hdiJobManagementClient.JobManagement.SubmitHiveJob(job);
var jobId = jobResponse.JobSubmissionJsonResponse.Id;
System.Console.WriteLine("Response status code is " + jobResponse.StatusCode);
System.Console.WriteLine("JobId is " + jobId);
System.Console.WriteLine("Waiting for the job completion ...");
// Wait for job completion
var jobDetail = hdiJobManagementClient.JobManagement.GetJob(jobId).JobDetail;
while (!jobDetail.Status.JobComplete)
{
Thread.Sleep(1000);
jobDetail = hdiJobManagementClient.JobManagement.GetJob(jobId).JobDetail;
}
// Get job output
var storageAccess = new AzureStorageAccess(storageAccount, storageKey,
storageContainer);
var output = (jobDetail.ExitValue == 0)
? hdiJobManagementClient.JobManagement.GetJobOutput(jobId, storageAccess) // fetch stdout output in case of success
: hdiJobManagementClient.JobManagement.GetJobErrorLogs(jobId, storageAccess); // fetch stderr output in case of failure
System.Console.WriteLine("Job output is: ");
using (var reader = new StreamReader(output, Encoding.UTF8))
{
string value = reader.ReadToEnd();
System.Console.WriteLine(value);
}
}
public InsightPhotoController(IHostingEnvironment hostingEnv)
{
_env = hostingEnv;
var clusterCredentials = new BasicAuthenticationCloudCredentials {
Username = ExistingClusterUsername, Password = ExistingClusterPassword
};
_jobClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials);
}
private JobGetResponse GetJobFinalStatus(HDInsightJobManagementClient client, string jobId)
{
var jobStatus = client.JobManagement.GetJob(jobId);
while (!jobStatus.JobDetail.Status.JobComplete)
{
jobStatus = client.JobManagement.GetJob(jobId);
// Optional to sleep here. Sleep for 1 sec instead of keep pooling.
Thread.Sleep(1000);
}
return(jobStatus);
}
static void Main(string[] args)
{
Console.WriteLine("The application is running ...");
var clusterCredentials = new BasicAuthenticationCloudCredentials {
Username = ExistingClusterUsername, Password = ExistingClusterPassword
};
_hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials);
SubmitMRJob();
Console.WriteLine("Press ENTER to continue ...");
Console.ReadLine();
}
public void CheckValidJobUserName()
{
var credentials = new BasicAuthenticationCredentials
{
UserName = "******",
Password = ""
};
var client = new HDInsightJobManagementClient("TestCluster", credentials);
Assert.Equal(credentials.UserName.ToLower(CultureInfo.CurrentCulture), client.Username);
client = new HDInsightJobManagementClient("TestCluster", credentials, new HttpClient());
Assert.Equal(credentials.UserName.ToLower(CultureInfo.CurrentCulture), client.Username);
}
public RecalculationJob(StorageConfiguration storageConfiguration, HDInsightConfiguration configuration)
{
_storageConfiguration = storageConfiguration;
_configuration = configuration;
_storageAccess = new AzureStorageAccess(
_configuration.DefaultStorageAccountName,
_configuration.DefaultStorageAccountKey,
_configuration.DefaultStorageContainerName);
var clusterCredentials = new BasicAuthenticationCloudCredentials {
Username = _configuration.ExistingClusterUsername, Password = _configuration.ExistingClusterPassword
};
_hdiJobManagementClient = new HDInsightJobManagementClient(_configuration.ExistingClusterUri, clusterCredentials);
}
static void Main(string[] args)
{
var key = new SecureString();
foreach (char c in secretKey)
{
key.AppendChar(c);
}
var tokenCreds = GetTokenCloudCredentials(tenantId, appId, key);
var resourceManagementClient = new ResourceManagementClient(new TokenCloudCredentials(subId.ToString(), tokenCreds.Token));
resourceManagementClient.Providers.Register("Microsoft.HDInsight");
var hdiManagementClient = new HDInsightManagementClient(new TokenCredentials(tokenCreds.Token));
hdiManagementClient.SubscriptionId = subId.ToString();
var clusterName = GetClusterName(hdiManagementClient);
Console.ReadLine();
var hdiJobManagementClient = new HDInsightJobManagementClient(clusterName + ".azurehdinsight.net",
new BasicAuthenticationCloudCredentials
{
Username = user,
Password = password
});
Dictionary <string, string> defines = new Dictionary <string, string> {
{ "hive.execution.engine", "tez" }, { "hive.exec.reducers.max", "1" }
};
List <string> hadoopArgs = new List <string> {
{ "argA" }, { "argB" }
};
var job = new HiveJobSubmissionParameters
{
Query = "SHOW TABLES",
Defines = defines,
Arguments = hadoopArgs
};
SubmitJobToCluster(hdiJobManagementClient, job);
Console.ReadLine();
}
public static void SubmitPigJob()
{
const string ExistingClusterName = "sparkdheetest";
const string ExistingClusterUri = ExistingClusterName + ".azurehdinsight.net";
const string ExistingClusterUsername = "******";
const string ExistingClusterPassword = "******";
var clusterCredentials = new BasicAuthenticationCloudCredentials {
Username = ExistingClusterUsername, Password = ExistingClusterPassword
};
HDInsightJobManagementClient _hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials);
////SubmitPigJob();
var parameters = new PigJobSubmissionParameters
{
//A = LOAD 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/information.txt' using PigStorage (‘\t’) as (FName: chararray, LName: chararray, MobileNo: chararray, City: chararray, Profession: chararray);
// B = FOREACH A generate FName,MobileNo,Profession;
//DUMP B;
//wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/sample.pig
//Query = @"PigSampleIn = LOAD 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/input.txt' USIG PigStorage(',') AS (ProfileID:chararray, SessionStart:chararray, Duration:int, SrcIPAddress:chararray, GameType:chararray);
// GroupProfile = Group PigSampleIn all;
// PigSampleOut = Foreach GroupProfile Generate PigSampleIn.ProfileID, SUM(PigSampleIn.Duration);
// Store PigSampleOut into 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/output.txt' USING PigStorage (',');"
// File =
};
Console.WriteLine("Submitting the Pig job to the cluster...");
var response = _hdiJobManagementClient.JobManagement.SubmitPigJob(parameters);
Console.WriteLine("Validating that the response is as expected...");
Console.WriteLine("Response status code is " + response.StatusCode);
Console.WriteLine("Validating the response object...");
Console.WriteLine("JobId is " + response.JobSubmissionJsonResponse.Id);
var jobs = _hdiJobManagementClient.JobManagement.ListJobs();
foreach (var job in jobs)
{
//var deails = _hdiJobManagementClient.JobManagement.GetJobOutput(job.Id)
}
Console.WriteLine("Press ENTER to continue ...");
Console.ReadLine();
}
public void CheckValidJobUserName()
{
using (var context = UndoContext.Current)
{
context.Start();
var credentials = new BasicAuthenticationCloudCredentials
{
Username = "******",
Password = ""
};
var client = new HDInsightJobManagementClient("TestCluster", credentials);
Assert.Equal(credentials.Username.ToLower(CultureInfo.CurrentCulture), client.UserName);
client = new HDInsightJobManagementClient("TestCluster", credentials, new HttpClient());
Assert.Equal(credentials.Username.ToLower(CultureInfo.CurrentCulture), client.UserName);
}
}
public static void SubmitPigJob(string filename)
{
const string ExistingClusterName = "orionml";
const string ExistingClusterUri = ExistingClusterName + ".azurehdinsight.net";
const string ExistingClusterUsername = "******";
const string ExistingClusterPassword = "******";
var clusterCredentials = new BasicAuthenticationCloudCredentials {
Username = ExistingClusterUsername, Password = ExistingClusterPassword
};
HDInsightJobManagementClient _hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials);
var parameters = new PigJobSubmissionParameters
{
File = "wasbs:///user/root/" + filename + ".pig"
};
Console.WriteLine("Submitting the Pig job with file name [" + filename + "] to the cluster...");
var response = _hdiJobManagementClient.JobManagement.SubmitPigJob(parameters);
Console.WriteLine("JobId is " + response.JobSubmissionJsonResponse.Id);
}
internal void WaitForJobCompletion(HDInsightJobManagementClient client, string jobId, TimeSpan pollingInterval, TimeSpan duration, bool cancelJob = false)
{
var startTime = DateTime.UtcNow;
var endTime = DateTime.UtcNow;
var jobDetail = client.JobManagement.GetJob(jobId);
while (!jobDetail.JobDetail.Status.JobComplete)
{
if (((endTime = DateTime.UtcNow) - startTime) > duration)
{
string exceptionMessage = string.Format(CultureInfo.InvariantCulture, "The requested task failed to complete in the allotted time ({0})", duration);;
if (cancelJob)
{
client.JobManagement.KillJob(jobId);
exceptionMessage = string.Format(CultureInfo.InvariantCulture, "{0} Killed the Job {1}", exceptionMessage, jobId);
}
throw new CloudException(exceptionMessage);
}
jobDetail = client.JobManagement.GetJob(jobId);
}
}
private JobGetResponse GetJobFinalStatus(HDInsightJobManagementClient client, string jobId)
{
var jobStatus = client.JobManagement.GetJob(jobId);
while (!jobStatus.JobDetail.Status.JobComplete)
{
jobStatus = client.JobManagement.GetJob(jobId);
// Optional to sleep here. Sleep for 1 sec instead of keep pooling.
Thread.Sleep(1000);
}
return jobStatus;
}
public static HDInsightJobManagementClient GetHDInsightJobManagementClient(string dnsName, BasicAuthenticationCloudCredentials creds)
{
var client = new HDInsightJobManagementClient(dnsName, creds);
return(AddMockHandler(ref client));
}
internal void WaitForJobCompletion(HDInsightJobManagementClient client, string jobId, TimeSpan pollingInterval, TimeSpan duration, bool cancelJob = false)
{
var startTime = DateTime.UtcNow;
var endTime = DateTime.UtcNow;
var jobDetail = client.JobManagement.GetJob(jobId);
while (!jobDetail.JobDetail.Status.JobComplete)
{
if (((endTime = DateTime.UtcNow) - startTime) > duration)
{
string exceptionMessage = string.Format(CultureInfo.InvariantCulture, "The requested task failed to complete in the allotted time ({0})", duration); ;
if (cancelJob)
{
client.JobManagement.KillJob(jobId);
exceptionMessage = string.Format(CultureInfo.InvariantCulture, "{0} Killed the Job {1}", exceptionMessage, jobId);
}
throw new CloudException(exceptionMessage);
}
jobDetail = client.JobManagement.GetJob(jobId);
}
}
