/// <summary>
/// Deserializes a jobDetails creation result object from a payload string.
/// </summary>
/// <param name="payload">
/// The payload.
/// </param>
/// <returns>
/// A JobCreationResults object representing the payload.
/// </returns>
public JobCreationResults DeserializeJobCreationResults(string payload)
{
JobCreationResults results = new JobCreationResults();
results.ErrorCode = string.Empty;
results.HttpStatusCode = HttpStatusCode.Accepted;
XmlDocumentConverter documentConverter = new XmlDocumentConverter();
var document = documentConverter.GetXmlDocument(payload);
DynaXmlNamespaceTable nameTable = new DynaXmlNamespaceTable(document);
var node = document.SelectSingleNode("//def:PassthroughResponse/def:Data", nameTable.NamespaceManager);
results.JobId = node.InnerText;
XmlElement error = (XmlElement)document.SelectSingleNode("//def:PassthroughResponse/def:Error", nameTable.NamespaceManager);
if (error.IsNotNull())
{
var errorId = error.SelectSingleNode("//def:ErrorId", nameTable.NamespaceManager);
if (errorId.IsNotNull())
{
results.ErrorCode = errorId.InnerText;
}
var statusCode = error.SelectSingleNode("//def:StatusCode", nameTable.NamespaceManager);
if (statusCode.IsNotNull())
{
HttpStatusCode httpStatusCode = HttpStatusCode.Accepted;
if (HttpStatusCode.TryParse(statusCode.InnerText, out httpStatusCode))
{
results.HttpStatusCode = httpStatusCode;
}
}
}
return results;
}
/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="job">The jobDetails to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>An awaitable task that represents the action.</returns>
public static async Task<JobDetails> WaitForJobCompletionAsync(
this IJobSubmissionClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
{
client.ArgumentNotNull("client");
job.ArgumentNotNull("jobDetails");
return await client.WaitForJobCompletionAsync(job.JobId, duration, cancellationToken);
}
/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="job">The jobDetails to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>An awaitable task that represents the action.</returns>
public static async Task <JobDetails> WaitForJobCompletionAsync(
this IJobSubmissionClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
{
client.ArgumentNotNull("client");
job.ArgumentNotNull("jobDetails");
return(await client.WaitForJobCompletionAsync(job.JobId, duration, cancellationToken));
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client)
{
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
public string SerializeJobCreationResults(JobCreationResults jobResults)
{
var result = new PassthroughResponse();
if (jobResults.ErrorCode.IsNotNullOrEmpty() || jobResults.HttpStatusCode != HttpStatusCode.Accepted)
{
result.Error = new PassthroughErrorResponse { StatusCode = jobResults.HttpStatusCode, ErrorId = jobResults.ErrorCode };
}
result.Data = jobResults.JobId;
return this.SerializeJobDetails(result);
}
public Task<JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters)
{
this.SubmitSqoopJobCalled = true;
var job = new JobCreationResults() { JobId = JobId };
return Task.Run(() => job);
}
public Task<JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
{
this.SubmitHiveJobCalled = true;
var job = new JobCreationResults() { JobId = JobId };
return Task.Run(() => job);
}
private JobCreationResults CreateJobSuccessResult(JobDetails jobDetailsHistoryEntry, string jobName)
{
if (this.cluster.IsNull())
{
throw new InvalidOperationException("The cluster could not be found.");
}
if (this.credentials.UserName != this.cluster.Cluster.HttpUserName && this.credentials.Password != this.cluster.Cluster.HttpPassword)
{
throw new UnauthorizedAccessException("The supplied credential do not have access to the server.");
}
lock (this.cluster.JobQueue)
{
this.LogMessage("Starting jobDetails '{0}'.", jobName);
var jobCreationResults = new JobCreationResults()
{
JobId = "job_" + Guid.NewGuid().ToString(),
HttpStatusCode = HttpStatusCode.OK
};
jobDetailsHistoryEntry.Name = jobName;
jobDetailsHistoryEntry.JobId = jobCreationResults.JobId;
jobDetailsHistoryEntry.PercentComplete = "map 0% reduce 0%";
this.cluster.JobQueue.Add(jobDetailsHistoryEntry.JobId, jobDetailsHistoryEntry);
return jobCreationResults;
}
}
public void ICanSerializeAndDeserialzeCreationResults()
{
JobCreationResults expected = new JobCreationResults() { HttpStatusCode = HttpStatusCode.Accepted, JobId = "job123" };
JobPayloadServerConverter ser = new JobPayloadServerConverter();
JobPayloadConverter deser = new JobPayloadConverter();
var payload = ser.SerializeJobCreationResults(expected);
var actual = deser.DeserializeJobCreationResults(payload);
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client)
{
// Wait for job completion, displaying progress
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
Console.Write(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(5));
Console.Write(".");
}
Console.WriteLine(jobInProgress.StatusCode);
}
/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="job">The jobDetails to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>The jobDetails's pigJobCreateParameters.</returns>
public static JobDetails WaitForJobCompletion(
this IJobSubmissionClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
{
return WaitForJobCompletionAsync(client, job, duration, cancellationToken).WaitForResult();
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client, String idUsuario, String subIdUsuario)
{
try
{
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName);
CloudBlockBlob blockBlob = container.GetBlockBlobReference("pruebaProgreso-" + idUsuario + "." + subIdUsuario);
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_RUNNING);
jobInProgress = client.GetJob(jobInProgress.JobId);
}
if (jobInProgress.StatusCode == JobStatusCode.Failed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_FAIL);
}
else if (jobInProgress.StatusCode == JobStatusCode.Completed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_SUCCESS);
}
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
throw;
}
finally {
}
}
/// <summary>
/// Method that waits for a jobDetails to complete.
/// </summary>
/// <param name="client">The Hadoop client to use.</param>
/// <param name="job">The jobDetails to wait for.</param>
/// <param name="duration">The duration to wait before timing out.</param>
/// <param name="cancellationToken">
/// The Cancellation Token for the request.
/// </param>
/// <returns>The jobDetails's pigJobCreateParameters.</returns>
public static JobDetails WaitForJobCompletion(
this IJobSubmissionClient client, JobCreationResults job, TimeSpan duration, CancellationToken cancellationToken)
{
return(WaitForJobCompletionAsync(client, job, duration, cancellationToken).WaitForResult());
}
