private JobCreationResults CreateJobSuccessResult(JobDetails jobDetailsHistoryEntry, string jobName)
{
if (this.cluster.IsNull())
{
throw new InvalidOperationException("The cluster could not be found.");
}
//if (this.credentials.UserName != this.cluster.Cluster.HttpUserName && this.credentials.Password != this.cluster.Cluster.HttpPassword)
//{
// throw new UnauthorizedAccessException("The supplied credential do not have access to the server.");
//}
lock (this.cluster.JobQueue)
{
this.LogMessage("Starting jobDetails '{0}'.", jobName);
var jobCreationResults = new JobCreationResults {
JobId = "job_" + Guid.NewGuid().ToString(), HttpStatusCode = HttpStatusCode.OK
};
jobDetailsHistoryEntry.Name = jobName;
jobDetailsHistoryEntry.JobId = jobCreationResults.JobId;
jobDetailsHistoryEntry.PercentComplete = "map 0% reduce 0%";
this.cluster.JobQueue.Add(jobDetailsHistoryEntry.JobId, jobDetailsHistoryEntry);
return(jobCreationResults);
}
}
public async Task ProcessRecord()
{
if (string.IsNullOrWhiteSpace(this.JobId) || string.IsNullOrWhiteSpace(this.Cluster))
{
this.JobId = this.Job.JobId;
this.Cluster = this.Job.Cluster;
}
IJobSubmissionClient client = this.GetClient(this.Cluster);
client.JobStatusEvent += this.ClientOnJobStatus;
JobDetails jobDetail = null;
if (ReduceWaitTime)
{
jobDetail = await client.GetJobAsync(this.JobId);
while (jobDetail.StatusCode != JobStatusCode.Completed && jobDetail.StatusCode != JobStatusCode.Failed &&
jobDetail.StatusCode != JobStatusCode.Canceled)
{
jobDetail = await client.GetJobAsync(this.JobId);
}
}
else
{
var job = new JobCreationResults {
JobId = this.JobId
};
jobDetail = await client.WaitForJobCompletionAsync(job, TimeSpan.FromSeconds(this.WaitTimeoutInSeconds), this.tokenSource.Token);
}
this.Output.Add(new AzureHDInsightJob(jobDetail, this.Cluster));
}
/// <summary>
/// Deserializes a jobDetails creation result object from a payload string.
/// </summary>
/// <param name="payload">
/// The payload.
/// </param>
/// <returns>
/// A JobCreationResults object representing the payload.
/// </returns>
public JobCreationResults DeserializeJobCreationResults(string payload)
{
JobCreationResults results = new JobCreationResults();
results.ErrorCode = string.Empty;
results.HttpStatusCode = HttpStatusCode.Accepted;
XmlDocumentConverter documentConverter = new XmlDocumentConverter();
var document = documentConverter.GetXmlDocument(payload);
DynaXmlNamespaceTable nameTable = new DynaXmlNamespaceTable(document);
var node = document.SelectSingleNode("//def:PassthroughResponse/def:Data", nameTable.NamespaceManager);
results.JobId = node.InnerText;
XmlElement error = (XmlElement)document.SelectSingleNode("//def:PassthroughResponse/def:Error", nameTable.NamespaceManager);
if (error.IsNotNull())
{
var errorId = error.SelectSingleNode("//def:ErrorId", nameTable.NamespaceManager);
if (errorId.IsNotNull())
{
results.ErrorCode = errorId.InnerText;
}
var statusCode = error.SelectSingleNode("//def:StatusCode", nameTable.NamespaceManager);
if (statusCode.IsNotNull())
{
HttpStatusCode httpStatusCode = HttpStatusCode.Accepted;
if (HttpStatusCode.TryParse(statusCode.InnerText, out httpStatusCode))
{
results.HttpStatusCode = httpStatusCode;
}
}
}
return(results);
}
public Task <JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters)
{
if (hiveJobCreateParameters.Query.IsNullOrEmpty())
{
hiveJobCreateParameters.File.ArgumentNotNullOrEmpty("File");
if (hiveJobCreateParameters.File.Contains("://") &&
!hiveJobCreateParameters.File.StartsWith("wasb", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Invalid file protocol : " + hiveJobCreateParameters.File);
}
}
else
{
this.PrepareQueryJob(hiveJobCreateParameters);
}
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails
{
Name = hiveJobCreateParameters.JobName,
Query = hiveJobCreateParameters.Query,
StatusDirectory = hiveJobCreateParameters.StatusFolder
},
hiveJobCreateParameters.JobName);
return(TaskEx2.FromResult(retval));
}
//Run Hive Job
public static void DoHiveOperations()
{
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "Show tables job",
StatusFolder = "/TableListFolder",
Query = "show tables;"
};
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName);
var jobClient = JobSubmissionClientFactory.Connect(creds);
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
Console.Write("Executing Hive Job.");
// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
// Print the Hive job output
System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId);
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
Console.Write("Done..List of Tables are:\n");
Console.WriteLine(reader.ReadToEnd());
}
/// <summary>
/// Performs HQL query and returns the query results.
/// </summary>
/// <param name="jobParams">The query parameters.</param>
/// <returns>The query result.</returns>
public string Query(HiveJobCreateParameters jobParams)
{
// Assign status folder
jobParams.StatusFolder = RootDirectory + "/status";
JobCreationResults jobDetails = null;
try
{
// Create Hive job
jobDetails = this.job.CreateHiveJob(jobParams);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while creating a Hive job\n" + e);
}
JobDetails jobInProgress = null;
try
{
// Get job status
jobInProgress = this.job.GetJob(jobDetails.JobId);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while getting Hive job status\n" + e);
}
// If job is not finished then sleep until the next client polling interval
while (jobInProgress.StatusCode != JobStatusCode.Completed &&
jobInProgress.StatusCode != JobStatusCode.Failed)
{
try
{
// Get job status
jobInProgress = this.job.GetJob(jobDetails.JobId);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while getting Hive job status\n" + e);
}
Thread.Sleep(this.client.PollingInterval);
}
try
{
// Job is finished; get its output stream, read it, and return the value
return(new StreamReader(this.job.GetJobOutput(jobDetails.JobId)).ReadToEnd());
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while reading Hibe job result\n" + e);
}
return(string.Empty);
}
public void ServiceHost_JobSubmissionRecieved(object sender, JobSubmissionMessage e)
{
Trace.WriteLine("JobSubmissionRecieved Recieved User Id : " + e.idUsuario, "Warning");
try
{
// Obtener la cuenta de almacenamiento
// Para actualizar metadatos
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName);
// Obtencion de variables para la conxion con el cluster
string subscriptionID = VariablesConfiguracion.subscriptionID;
string certFriendlyName = VariablesConfiguracion.certFriendlyName;
string clusterName = VariablesConfiguracion.clusterName;
// Definicion de la tarea MapReduce
MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
{
JarFile = "wasb:///CienciaCelularMR.jar",
ClassName = "Main",
StatusFolder = "wasb:///scicluster/test/status-" + e.idUsuario + "." + e.subIdUsuario,
};
mrJobDefinition.Arguments.Add("wasb:///" + e.nomEntrada);
mrJobDefinition.Arguments.Add("wasb:///scicluster/test/output-" + e.idUsuario + "." + e.subIdUsuario);
// Obtener el objeto certificate
X509Store store = new X509Store();
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = FindCertificate(StoreLocation.CurrentUser, StoreName.My, X509FindType.FindByThumbprint, VariablesConfiguracion.thumbprint);
JobSubmissionCertificateCredential creds = new JobSubmissionCertificateCredential(new Guid(subscriptionID), cert, clusterName);
// Se crea un cliente Hadoop para conectarse con HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
//Actualizacion de metadatos
CloudBlockBlob infoSimulation = container.GetBlockBlobReference(VariablesConfiguracion.infoSimulation + "-" + e.idUsuario);
infoSimulation.UploadText(VariablesConfiguracion.JOB_STARTING);
// Se lanza la ejecucion de los jobs MapReduce
JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition);
// Esperar hasta que finalice la ejecucion
WaitForJobCompletion(mrJobResults, jobClient, e.idUsuario, e.subIdUsuario);
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
throw;
}
}
private static void WaitForJobCompletion(IJobSubmissionClient client, JobCreationResults jobResults)
{
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(10));
}
}
public Task <JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
{
this.SubmitHiveJobCalled = true;
var job = new JobCreationResults()
{
JobId = JobId
};
return(Task.Run(() => job));
}
public Task <JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters)
{
this.SubmitSqoopJobCalled = true;
var job = new JobCreationResults()
{
JobId = JobId
};
return(Task.Run(() => job));
}
public void ICanSerializeAndDeserialzeCreationResults()
{
JobCreationResults expected = new JobCreationResults()
{
HttpStatusCode = HttpStatusCode.Accepted, JobId = "job123"
};
JobPayloadServerConverter ser = new JobPayloadServerConverter();
JobPayloadConverter deser = new JobPayloadConverter();
var payload = ser.SerializeJobCreationResults(expected);
var actual = deser.DeserializeJobCreationResults(payload);
}
internal AzureYarnJob(AzureYarnClient client, Uri errorLocation, Uri outputDir, JobCreationResults launchJob)
{
this.client = client;
this.errorLocation = errorLocation;
this.outputDir = outputDir;
this.launchJob = launchJob;
this.status = JobStatus.Waiting;
this.completion = new TaskCompletionSource <JobStatus>();
this.started = new TaskCompletionSource <bool>();
Task.Run(() => MonitorJob());
}
public string SerializeJobCreationResults(JobCreationResults jobResults)
{
var result = new PassthroughResponse();
if (jobResults.ErrorCode.IsNotNullOrEmpty() || jobResults.HttpStatusCode != HttpStatusCode.Accepted)
{
result.Error = new PassthroughErrorResponse {
StatusCode = jobResults.HttpStatusCode, ErrorId = jobResults.ErrorCode
};
}
result.Data = jobResults.JobId;
return(this.SerializeJobDetails(result));
}
private static JobCreationResults RunHiveJob(HiveJobCreateParameters job)
{
JobCreationResults jobDetails = jobSubmissionClient.CreateHiveJob(job);
JobDetails jobInProgress = jobSubmissionClient.GetJob(jobDetails.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = jobSubmissionClient.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromMilliseconds(IHadoopClientExtensions.GetPollingInterval()));
}
Assert.IsNull(jobDetails.ErrorCode, "Should not fail hive jobDetails submission");
Assert.IsNotNull(jobDetails.JobId, "Should have a non-null jobDetails id");
return(jobDetails);
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client)
{
// Wait for job completion, displaying progress
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
Console.Write(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(5));
Console.Write(".");
}
Console.WriteLine(jobInProgress.StatusCode);
}
public Task <JobCreationResults> CreateSqoopJobAsync(SqoopJobCreateParameters sqoopJobCreateParameters)
{
if (sqoopJobCreateParameters == null)
{
throw new ArgumentNullException("sqoopJobCreateParameters");
}
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails {
Query = sqoopJobCreateParameters.Command, StatusDirectory = sqoopJobCreateParameters.StatusFolder
}, string.Empty);
return(TaskEx2.FromResult(retval));
}
//Helper Function to Wait while job executes
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client)
{
Trace.WriteLine("Entering WaitForJobCompletion method");
Trace.TraceInformation("Executing WaitForJobCompletion method " + DateTime.Now.ToLongTimeString());
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed &&
jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = client.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromSeconds(1));
Console.Write(".");
}
Trace.WriteLine("Leaving WaitForJobCompletion method");
}
public Task <JobCreationResults> CreatePigJobAsync(PigJobCreateParameters pigJobCreateParameters)
{
if (pigJobCreateParameters == null)
{
throw new ArgumentNullException("pigJobCreateParameters");
}
this.PrepareQueryJob(pigJobCreateParameters);
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails {
Query = pigJobCreateParameters.Query, StatusDirectory = pigJobCreateParameters.StatusFolder
}, string.Empty);
return(TaskEx2.FromResult(retval));
}
private void RoundtripAvroDataToHive()
{
this.UploadFiles();
string testDirectory = string.Format(
"{0}{1}@{2}/{3}",
Constants.WabsProtocolSchemeName,
this.storageAccount.Container,
this.storageAccount.Name,
AvroIntegrationWithHiveConfigurations.TestRootDirectory);
RunHiveJob(new HiveJobCreateParameters {
File = testDirectory + "/query.hql", JobName = "CreateLargeClassTable"
});
JobCreationResults result =
RunHiveJob(
new HiveJobCreateParameters
{
Query = "SELECT BoolMember, DateTimeMember, DecimalMember, DoubleMember, EnumMember, FloatMember, base64(GuidMember), IntArrayMember, IntListMember, IntMapMember, IntMember, LongMember, SByteArrayMember, StringMember FROM " + AvroIntegrationWithHiveConfigurations.HiveTableName,
JobName = "QueryLargeClassTable",
StatusFolder = "/" + AvroIntegrationWithHiveConfigurations.QueryResultFolder
});
Stream stream = jobSubmissionClient.GetJobOutput(result.JobId);
this.dataProvider.CompareToQueryResult(new StreamReader(stream).ReadToEnd());
RunHiveJob(new HiveJobCreateParameters {
File = testDirectory + "/extraquery.hql", JobName = "CreateExtraLargeClassTable"
});
RunHiveJob(
new HiveJobCreateParameters
{
Query =
"INSERT INTO TABLE " + AvroIntegrationWithHiveConfigurations.HiveExtraTableName + " SELECT * FROM " +
AvroIntegrationWithHiveConfigurations.HiveTableName + ";"
});
Task <Stream> task = this.storageClient.Read(new Uri(testDirectory + "/extradata/000000_0"));
task.Wait();
Stream streamResult = task.Result;
this.dataProvider.CompareToAvroFile(streamResult);
}
//Run Sample Map Reduce Job
public static void DoMapReduce()
{
// Define the MapReduce job
MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
{
JarFile = "wasb:///example/jars/hadoop-examples.jar",
ClassName = "wordcount"
};
mrJobDefinition.Arguments.Add("wasb:///example/data/gutenberg/davinci.txt");
mrJobDefinition.Arguments.Add("wasb:///example/data/WordCountOutput");
//Get certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName);
// Create a hadoop client to connect to HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
// Run the MapReduce job
JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition);
Console.Write("Executing WordCount MapReduce Job.");
// Wait for the job to complete
WaitForJobCompletion(mrJobResults, jobClient);
// Print the MapReduce job output
Stream stream = new MemoryStream();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=" + Constants.storageAccount + ";AccountKey=" + Constants.storageAccountKey);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(Constants.container);
CloudBlockBlob blockBlob = blobContainer.GetBlockBlobReference("example/data/WordCountOutput/part-r-00000");
blockBlob.DownloadToStream(stream);
stream.Position = 0;
StreamReader reader = new StreamReader(stream);
Console.Write("Done..Word counts are:\n");
Console.WriteLine(reader.ReadToEnd());
}
private static void WaitForJobCompletion(JobCreationResults jobResults, IJobSubmissionClient client, String idUsuario, String subIdUsuario)
{
try
{
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName);
CloudBlockBlob blockBlob = container.GetBlockBlobReference("pruebaProgreso-" + idUsuario + "." + subIdUsuario);
JobDetails jobInProgress = client.GetJob(jobResults.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_RUNNING);
jobInProgress = client.GetJob(jobInProgress.JobId);
}
if (jobInProgress.StatusCode == JobStatusCode.Failed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_FAIL);
}
else if (jobInProgress.StatusCode == JobStatusCode.Completed)
{
blockBlob.UploadText(VariablesConfiguracion.JOB_SUCCESS);
}
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
throw;
}
finally {
}
}
private void InitializeSimulator()
{
if (this.cluster.IsNull())
{
throw new InvalidOperationException("Cluster is not set to a valid instance.");
}
JobCreationResults piJob =
this.CreateMapReduceJob(
new MapReduceJobCreateParameters
{
ClassName = "pi",
JarFile = "wasb://containerName@hostname/jarfileName",
JobName = "pi estimation jobDetails",
StatusFolder = "/pioutput"
});
JobCreationResults hiveJob =
this.CreateHiveJob(
new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables", StatusFolder = "/hiveoutput"
});
JobCreationResults pigJob = this.CreatePigJob(new PigJobCreateParameters {
Query = "show tables", StatusFolder = "/pigoutput"
});
this.WriteJobOutput(piJob.JobId, "/pioutput", JobSuccesful);
this.WriteJobOutput(hiveJob.JobId, "/hiveoutput", "hivesampletable");
this.WriteJobOutput(pigJob.JobId, "/pigoutput", JobSuccesful);
this.WriteJobError(piJob.JobId, "/pioutput", JobSuccesful);
this.WriteJobError(hiveJob.JobId, "/hiveoutput", JobSuccesful);
this.WriteJobError(pigJob.JobId, "/pigoutput", JobSuccesful);
this.WriteJobLogSummary("/pioutput", piJob.JobId);
this.WriteJobLogSummary("/hiveoutput", hiveJob.JobId);
this.WriteJobLogSummary("/pigoutput", pigJob.JobId);
}
private static void SubmitJobs()
{
// Get HDInsight cluster configuration settings
string clusterName = ConfigurationManager.AppSettings["ClusterName"];
string userName = ConfigurationManager.AppSettings["UserName"];
string password = ConfigurationManager.AppSettings["Password"];
// Create basic authentication credential for cluster
BasicAuthCredential bcred = new BasicAuthCredential();
bcred.Server = new Uri("https://" + clusterName + ".azurehdinsight.net");
bcred.UserName = userName;
bcred.Password = password;
// Create and submit Pig job
PigJobCreateParameters pigJob = new PigJobCreateParameters()
{
StatusFolder = "/data/racecar/scripts/processdatastatus",
File = "/data/racecar/scripts/processdata.pig"
};
var pigJobClient = JobSubmissionClientFactory.Connect(bcred);
JobCreationResults pigJobResults = pigJobClient.CreatePigJob(pigJob);
WaitForJobCompletion(pigJobResults, pigJobClient);
// Create and submit Hive job
HiveJobCreateParameters hiveJob = new HiveJobCreateParameters()
{
JobName = "Create Hive tables",
StatusFolder = "/data/racecar/scripts/createtablestatus",
File = "/data/racecar/scripts/createtables.hql"
};
var hiveJobClient = JobSubmissionClientFactory.Connect(bcred);
JobCreationResults hiveJobResults = hiveJobClient.CreateHiveJob(hiveJob);
WaitForJobCompletion(hiveJobResults, hiveJobClient);
}
/// <inheritdoc />
private async Task DownloadApplicationLogsAsync(string applicationId, string applicationUser, string containerId, string nodeId, string targetDirectory)
{
applicationId.ArgumentNotNullOrEmpty("applicationId");
applicationUser.ArgumentNotNullOrEmpty("applicationUser");
targetDirectory.ArgumentNotNullOrEmpty("targetDirectory");
if (!string.IsNullOrEmpty(containerId) && string.IsNullOrEmpty(nodeId))
{
throw new ArgumentException("NodeId was null or empty. If container id is specified, node id should also be specified");
}
if (!Directory.Exists(targetDirectory))
{
throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "The specified directory {0} does not exist.", targetDirectory));
}
var jobSubmissionClient = JobSubmissionClientFactory.Connect(this.HttpCredentials, this.customUserAgent);
var storageClient = ServiceLocator.Instance.Locate <IWabStorageAbstractionFactory>().Create(this.DefaultStorageCredentials);
// Check existence of application logs in the default storage account
Uri appLogContainer = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}/app-logs/{3}/logs/{4}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, applicationUser, applicationId));
var logFiles = await storageClient.List(appLogContainer, false);
if (!logFiles.Any())
{
throw new InvalidOperationException(string.Format("No logs found for application id {0}, user {1}, on cluster {2} at location {3}", applicationId, applicationUser, this.Cluster.Name, appLogContainer.AbsoluteUri));
}
// Application logs exist!
// Convert them to plain text by running YARN CLI
string jobName = string.Format("yarnlogs-{0}", Guid.NewGuid());
string statusFolderName = string.Format("/{0}", jobName);
string optionalContainerArguments = !string.IsNullOrEmpty(containerId) ? string.Format(" -containerId {0} -nodeAddress {1}", containerId, nodeId) : string.Empty;
string command = "";
if (this.Cluster.OSType == OSType.Windows)
{
command = string.Format("!cmd.exe /c yarn logs -applicationId {0} -appOwner {1}{2};", applicationId, applicationUser, optionalContainerArguments);
}
else if (this.Cluster.OSType == OSType.Linux)
{
command = string.Format("!yarn logs -applicationId {0} -appOwner {1}{2};", applicationId, applicationUser, optionalContainerArguments);
}
else
{
throw new NotSupportedException(String.Format("This functionality is not supported on clusters with OS Type: {0}", this.Cluster.OSType));
}
string queryFileName = string.Format("/{0}.hql", jobName);
Uri queryFileUri = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}{3}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, queryFileName));
Uri statusFolderUri = new Uri(string.Format(CultureInfo.InvariantCulture, "{0}{1}@{2}{3}", Constants.WabsProtocolSchemeName, this.DefaultStorageCredentials.ContainerName, this.DefaultStorageCredentials.Name, statusFolderName));
try
{
var bytes = Encoding.UTF8.GetBytes(command);
using (var memoryStream = new MemoryStream(bytes))
{
await storageClient.Write(queryFileUri, memoryStream);
}
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = jobName,
StatusFolder = statusFolderName,
File = queryFileName
};
JobCreationResults jobResults = jobSubmissionClient.CreateHiveJob(hiveJobDefinition);
WaitForJobCompletion(jobSubmissionClient, jobResults);
Uri logContentsFileUri = new Uri(string.Format("{0}/stdout", statusFolderUri.AbsoluteUri));
if (await storageClient.Exists(logContentsFileUri))
{
// create local file in the targetdirectory.
var localFilePath = Path.Combine(targetDirectory, string.Format("{0}_{1}.txt", this.Cluster.Name, string.IsNullOrEmpty(containerId) ? applicationId : containerId));
await storageClient.DownloadToFile(logContentsFileUri, localFilePath);
}
else
{
throw new InvalidOperationException(string.Format(
CultureInfo.InvariantCulture,
"Could not retrive logs for application id {0}, user {1} on cluster {2} at location {3}.",
applicationId,
applicationUser,
this.Cluster.Name,
appLogContainer.AbsoluteUri));
}
}
finally
{
// Cleanup what we created
if (storageClient.Exists(queryFileUri).WaitForResult())
{
storageClient.Delete(queryFileUri);
}
if (storageClient.Exists(statusFolderUri).WaitForResult())
{
storageClient.Delete(statusFolderUri);
}
}
}
public string HiveOutput(string q)
{
Trace.WriteLine("Entering HiveOutput method");
Trace.TraceInformation("Executing HiveOutput method at " + DateTime.Now.ToLongTimeString());
//Defining MapReduce Job
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "job",
StatusFolder = "/TableListFolder",
Query = q
};
Guid subscriptionId = new Guid("44fbb137-edbb-4044-9db9-0e1333e137cf"); //your-subscription-id
string clusterName = "tbanihumcluster";
// Get the certificate object from certificate store using the friendly name to identify it
X509Store store = new X509Store(StoreName.My);
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = store.Certificates.Cast <X509Certificate2>().First(item => item.FriendlyName == "Azdem187U23713U-1-8-2015-credentials");
var creds = new JobSubmissionCertificateCredential(subscriptionId, cert, clusterName);
// Create a hadoop client to connect to HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
// Run the MapReduce job
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
// Hive job output
System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId);
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
string value = reader.ReadToEnd();
value = value.Replace('\t', ',');
// Retrieve storage account from connection string.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference("mycontainer");
container.CreateIfNotExists();
// Retrieve reference to a blob named "myblob".
CloudBlockBlob blockBlob = container.GetBlockBlobReference("myblob");
blockBlob.UploadText(value);
Trace.WriteLine("Leaving HiveOutput method");
return(value);
}
public override async Task EndProcessing()
{
this.JobDefinition.ArgumentNotNull("JobDefinition");
IJobSubmissionClient client = this.GetClient(this.Cluster);
JobCreationResults jobCreationResults = null;
if (this.JobDefinition.StatusFolder.IsNullOrEmpty())
{
this.JobDefinition.StatusFolder = Guid.NewGuid().ToString();
}
var azureMapReduceJobDefinition = this.JobDefinition as AzureHDInsightMapReduceJobDefinition;
var azureHiveJobDefinition = this.JobDefinition as AzureHDInsightHiveJobDefinition;
var azurePigJobDefinition = this.JobDefinition as AzureHDInsightPigJobDefinition;
var azureStreamingJobDefinition = this.JobDefinition as AzureHDInsightStreamingMapReduceJobDefinition;
var azureSqoopJobDefinition = this.JobDefinition as AzureHDInsightSqoopJobDefinition;
if (azureMapReduceJobDefinition != null)
{
if (azureMapReduceJobDefinition.JobName.IsNullOrEmpty())
{
azureMapReduceJobDefinition.JobName = azureMapReduceJobDefinition.ClassName;
}
jobCreationResults = await SubmitMapReduceJob(azureMapReduceJobDefinition, client);
}
else if (azureHiveJobDefinition != null)
{
if (azureHiveJobDefinition.JobName.IsNullOrEmpty())
{
azureHiveJobDefinition.JobName = string.Format(
CultureInfo.InvariantCulture,
HiveJobNameFormat,
GetJobNameFromQueryOrFile(azureHiveJobDefinition.Query, azureHiveJobDefinition.File));
}
jobCreationResults = await SubmitHiveJob(azureHiveJobDefinition, client);
}
else if (azurePigJobDefinition != null)
{
jobCreationResults = await SubmitPigJob(azurePigJobDefinition, client);
}
else if (azureStreamingJobDefinition != null)
{
if (azureStreamingJobDefinition.JobName.IsNullOrEmpty())
{
azureStreamingJobDefinition.JobName = GetLastSegment(azureStreamingJobDefinition.Mapper);
}
jobCreationResults = await CreateStreamingJob(azureStreamingJobDefinition, client);
}
else if (azureSqoopJobDefinition != null)
{
jobCreationResults = await CreateSqoopJob(azureSqoopJobDefinition, client);
}
else
{
throw new NotSupportedException(
string.Format(CultureInfo.InvariantCulture, "Cannot start jobDetails of type : {0}.", this.JobDefinition.GetType()));
}
var startedJob = await client.GetJobAsync(jobCreationResults.JobId);
if (startedJob.ErrorCode.IsNotNullOrEmpty())
{
throw new InvalidOperationException("Failed to start jobDetails :" + startedJob.ErrorCode);
}
var jobDetail = new AzureHDInsightJob(startedJob, this.Cluster);
this.Output.Add(jobDetail);
}
private JobCreationResults CreateJobSuccessResult(JobDetails jobDetailsHistoryEntry, string jobName)
{
if (this.cluster.IsNull())
{
throw new InvalidOperationException("The cluster could not be found.");
}
//if (this.credentials.UserName != this.cluster.Cluster.HttpUserName && this.credentials.Password != this.cluster.Cluster.HttpPassword)
//{
// throw new UnauthorizedAccessException("The supplied credential do not have access to the server.");
//}
lock (this.cluster.JobQueue)
{
this.LogMessage("Starting jobDetails '{0}'.", jobName);
var jobCreationResults = new JobCreationResults { JobId = "job_" + Guid.NewGuid().ToString(), HttpStatusCode = HttpStatusCode.OK };
jobDetailsHistoryEntry.Name = jobName;
jobDetailsHistoryEntry.JobId = jobCreationResults.JobId;
jobDetailsHistoryEntry.PercentComplete = "map 0% reduce 0%";
this.cluster.JobQueue.Add(jobDetailsHistoryEntry.JobId, jobDetailsHistoryEntry);
return jobCreationResults;
}
}
