// [Fact]
public void GetPigJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new PigJobSubmissionParameters()
{
UserName = username,
Query = "records = LOAD '/example/pig/sahara-paleo-fauna.txt' AS (DateBP:int, Loc:chararray, Coordinates:chararray, Samples:chararray, Country:chararray, Laboratory:chararray);" +
"filtered_records = FILTER records by Country == 'Egypt' OR Country == 'Morocco';" +
"grouped_records = GROUP filtered_records BY Country;" +
"DUMP grouped_records;"
};
var jobid = client.JobManagement.SubmitPigJob(parameters).JobSubmissionJsonResponse.Id;
var response = client.JobManagement.GetJob(jobid);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
}
}
public virtual JobSubmissionResponse SubmitPigJob(AzureHDInsightPigJobDefinition pigJobDef)
{
var pigJobParams = new PigJobSubmissionParameters
{
Arguments = pigJobDef.Arguments,
Files = pigJobDef.Files,
StatusDir = pigJobDef.StatusFolder,
File = pigJobDef.File,
Query = pigJobDef.Query
};
return(HdInsightJobManagementClient.JobManagement.SubmitPigJob(pigJobParams));
}
public virtual JobSubmissionResponse SubmitPigJob(AzureHDInsightPigJobDefinition pigJobDef)
{
var pigJobParams = new PigJobSubmissionParameters
{
Arguments = ConvertListToString(pigJobDef.Arguments, "arg"),
Files = ConvertListToString(pigJobDef.Files, "file"),
StatusDir = pigJobDef.StatusFolder,
File = pigJobDef.File,
Query = pigJobDef.Query,
UserName = HdInsightJobManagementClient.Credentials.Username
};
return(HdInsightJobManagementClient.JobManagement.SubmitPigJob(pigJobParams));
}
public static void SubmitPigJob()
{
const string ExistingClusterName = "sparkdheetest";
const string ExistingClusterUri = ExistingClusterName + ".azurehdinsight.net";
const string ExistingClusterUsername = "******";
const string ExistingClusterPassword = "******";
var clusterCredentials = new BasicAuthenticationCloudCredentials {
Username = ExistingClusterUsername, Password = ExistingClusterPassword
};
HDInsightJobManagementClient _hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials);
////SubmitPigJob();
var parameters = new PigJobSubmissionParameters
{
//A = LOAD 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/information.txt' using PigStorage (‘\t’) as (FName: chararray, LName: chararray, MobileNo: chararray, City: chararray, Profession: chararray);
// B = FOREACH A generate FName,MobileNo,Profession;
//DUMP B;
//wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/sample.pig
//Query = @"PigSampleIn = LOAD 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/input.txt' USIG PigStorage(',') AS (ProfileID:chararray, SessionStart:chararray, Duration:int, SrcIPAddress:chararray, GameType:chararray);
// GroupProfile = Group PigSampleIn all;
// PigSampleOut = Foreach GroupProfile Generate PigSampleIn.ProfileID, SUM(PigSampleIn.Duration);
// Store PigSampleOut into 'wasb://clustercreatetemplate.blob.core.windows.net/clustercreatetemplate/output.txt' USING PigStorage (',');"
// File =
};
Console.WriteLine("Submitting the Pig job to the cluster...");
var response = _hdiJobManagementClient.JobManagement.SubmitPigJob(parameters);
Console.WriteLine("Validating that the response is as expected...");
Console.WriteLine("Response status code is " + response.StatusCode);
Console.WriteLine("Validating the response object...");
Console.WriteLine("JobId is " + response.JobSubmissionJsonResponse.Id);
var jobs = _hdiJobManagementClient.JobManagement.ListJobs();
foreach (var job in jobs)
{
//var deails = _hdiJobManagementClient.JobManagement.GetJobOutput(job.Id)
}
Console.WriteLine("Press ENTER to continue ...");
Console.ReadLine();
}
public void SubmitPigJobAndValidateOutput(PigJobSubmissionParameters parameters, bool runAyncAPI = false, bool isWindowsCluster = false)
{
using (var context = UndoContext.Current)
{
context.Start();
var client = TestUtils.GetHDInsightJobManagementClient(isWindowsCluster);
var response = runAyncAPI ? client.JobManagement.SubmitPigJobAsync(parameters).Result
: client.JobManagement.SubmitPigJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
WaitForJobCompletion(client, jobId, TestUtils.JobPollInterval, TestUtils.JobWaitInterval);
var jobStatus = client.JobManagement.GetJob(jobId);
var storageAccess = GetStorageAccessObject(isWindowsCluster);
if (jobStatus.JobDetail.ExitValue == 0)
{
// Retrieve Job Output
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Contains("(DEBUG,"));
}
}
else
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
private void SubmitPigJobAndValidateOutput(PigJobSubmissionParameters parameters, bool runAyncAPI = false)
{
using (var context = MockContext.Start(this.GetType()))
using (var client = this.CommonData.GetHDInsightJobClient(context))
{
var response = runAyncAPI ? client.Job.SubmitPigJobAsync(parameters).Result
: client.Job.SubmitPigJob(parameters);
Assert.NotNull(response);
var jobId = response.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = client.Job.WaitForJobCompletion(jobId, this.CommonData.JobWaitInterval, this.CommonData.JobPollInterval);
var storageAccess = GetStorageAccessObject();
if (jobStatus.ExitValue == 0)
{
// Retrieve Job Output
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
var output = client.Job.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.Contains("(DEBUG,", textOutput);
}
}
else
{
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
var output = client.Job.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
public static void SubmitPigJob(string filename)
{
const string ExistingClusterName = "orionml";
const string ExistingClusterUri = ExistingClusterName + ".azurehdinsight.net";
const string ExistingClusterUsername = "******";
const string ExistingClusterPassword = "******";
var clusterCredentials = new BasicAuthenticationCloudCredentials {
Username = ExistingClusterUsername, Password = ExistingClusterPassword
};
HDInsightJobManagementClient _hdiJobManagementClient = new HDInsightJobManagementClient(ExistingClusterUri, clusterCredentials);
var parameters = new PigJobSubmissionParameters
{
File = "wasbs:///user/root/" + filename + ".pig"
};
Console.WriteLine("Submitting the Pig job with file name [" + filename + "] to the cluster...");
var response = _hdiJobManagementClient.JobManagement.SubmitPigJob(parameters);
Console.WriteLine("JobId is " + response.JobSubmissionJsonResponse.Id);
}
/// <summary>
/// Submits a Pig job to an HDInsight cluster.
/// </summary>
/// <param name='parameters'>
/// Required. Pig job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public async Task <JobSubmissionResponse> SubmitPigJobAsync(PigJobSubmissionParameters parameters)
{
return(await SubmitPigJobAsync(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() }, CancellationToken.None));
}
/// <summary>
/// Submits a Pig job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Pig job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static async Task <JobSubmissionJsonResponse> SubmitPigJobAsync(this IJobOperations operations, PigJobSubmissionParameters parameters, CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.SubmitPigJobWithHttpMessagesAsync(parameters, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
/// <summary>
/// Submits a Pig job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Pig job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionJsonResponse SubmitPigJob(this IJobOperations operations, PigJobSubmissionParameters parameters)
{
return(operations.SubmitPigJobAsync(parameters).GetAwaiter().GetResult());
}
/// <summary>
/// Submits a Pig job to an HDInsight cluster.
/// </summary>
/// <param name='parameters'>
/// Required. Pig job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public async Task <AzureOperationResponse <JobSubmissionJsonResponse> > SubmitPigJobWithHttpMessagesAsync(PigJobSubmissionParameters parameters, Dictionary <string, List <string> > customHeaders = null, CancellationToken cancellationToken = default(CancellationToken))
{
using (var requestContents = new MemoryStream(Encoding.UTF8.GetBytes(parameters.GetJobPostRequestContent())))
{
return(await SubmitPigJobWithHttpMessagesAsync(requestContents, customHeaders, cancellationToken).ConfigureAwait(false));
}
}
/// <summary>
/// Submits an Hive job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Pig job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static Task <JobSubmissionResponse> SubmitPigJobAsync(this IJobOperations operations, PigJobSubmissionParameters parameters)
{
return(operations.SubmitPigJobAsync(parameters, CancellationToken.None));
}
/// <summary>
/// Submits an Hive job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Pig job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitPigJob(this IJobOperations operations, PigJobSubmissionParameters parameters)
{
return(Task.Factory.StartNew((object s) =>
{
return ((IJobOperations)s).SubmitPigJobAsync(parameters);
}
, operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult());
}
public void SubmitPigJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials()
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new PigJobSubmissionParameters()
{
Query = "LOGS = LOAD 'wasb:///example/data/sample.log';" +
"LEVELS = foreach LOGS generate REGEX_EXTRACT($0, '(TRACE|DEBUG|INFO|WARN|ERROR|FATAL)', 1) as LOGLEVEL;" +
"FILTEREDLEVELS = FILTER LEVELS by LOGLEVEL is not null;" +
"GROUPEDLEVELS = GROUP FILTEREDLEVELS by LOGLEVEL;" +
"FREQUENCIES = foreach GROUPEDLEVELS generate group as LOGLEVEL, COUNT(FILTEREDLEVELS.LOGLEVEL) as COUNT;" +
"RESULT = order FREQUENCIES by COUNT desc;" +
"DUMP RESULT;"
};
var response = client.JobManagement.SubmitPigJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = GetJobFinalStatus(client, jobId);
var storageAccess = GetStorageAccessObject();
if (jobStatus.JobDetail.ExitValue == 0)
{
// Retrieve Job Output
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Contains("(DEBUG,"));
}
}
else
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
/// <summary>
/// Submits a Pig job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Pig job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitPigJob(this IJobOperations operations, PigJobSubmissionParameters parameters)
{
return(operations.SubmitPigJob(new JobSubmissionParameters {
Content = parameters.GetJobPostRequestContent()
}));
}
