public static void CreateTables(JobSubmissionCertificateCredential creds, AzureSettings settings)
{
var storagePath = String.Format("wasb://{0}@{1}.blob.core.windows.net/", settings.ClusterName, settings.StorageAccount);
var TweetInPath = storagePath + @"/Tweets";
var IdentifiersInPath = storagePath + @"/Identifiers";
var hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "Create external tables",
StatusFolder = "/AAACreateTables",
Query = "DROP TABLE tweets; CREATE EXTERNAL TABLE tweets( id_str string, created_at string, retweet_count string, tweetText string, userName string, userId string, screenName string, countryCode string, placeType string, placeName string, placeType1 string, coordinates array<string>)" +
"ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' COLLECTION ITEMS TERMINATED BY ',' STORED AS TEXTFILE location '" + TweetInPath + "';" +
"DROP TABLE identifiers; CREATE EXTERNAL TABLE identifiers(identifier string)" +
"ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE location '" + IdentifiersInPath + "';"
};
// Submit the Hive job
var jobClient = JobSubmissionClientFactory.Connect(creds);
var jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
WaitForJobCompletion(jobResults, jobClient);
// Print the Hive job output
var stream = jobClient.GetJobOutput(jobResults.JobId);
var reader = new StreamReader(stream);
Console.WriteLine(reader.ReadToEnd());
}
private static JobSubmissionCertificateCredential GenerateJobSubmissionCert(AzureSettings settings)
{
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast<X509Certificate2>().First(item => item.FriendlyName == settings.LocalCertName);
var creds = new JobSubmissionCertificateCredential(new Guid(settings.SuscriptionId), cert, settings.ClusterName);
return creds;
}
public static string SubmitJob(HiveJobCreateParameters hiveJobDefinition, Func<Stream, string> submitCallback, string jobid = "")
{
string msg = string.Empty;
var start = DateTime.Now;
Console.WriteLine("开始提交job：" + hiveJobDefinition.JobName);
UBA.Common.LogHelperNet.Info("开始提交job：" + hiveJobDefinition.JobName, null);
string pfx = AppDomain.CurrentDomain.BaseDirectory + "ubaClient.pfx";
string subscriptionid = System.Configuration.ConfigurationSettings.AppSettings["Subscriptionid"];
string clustername = System.Configuration.ConfigurationSettings.AppSettings["Clustername"];
System.IO.Stream stream = null;
try
{
X509Certificate2 cert = new X509Certificate2(pfx, "1");//c8321a5a-6f7e-4f2e-a0c8-7b19f076877a
JobSubmissionCertificateCredential creds = new JobSubmissionCertificateCredential(new Guid(subscriptionid), cert, clustername, new Uri("https://management.core.chinacloudapi.cn"));
// Submit the Hive job
var jobClient = JobSubmissionClientFactory.Connect(creds);
if (!string.IsNullOrEmpty(jobid))
{
stream = jobClient.GetJobOutput(jobid);
}
else
{
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
msg = string.Format("提交job成功,耗时{0}秒\r\n开始处理job", DateTime.Now.Subtract(start).TotalMilliseconds / 1000);
Console.WriteLine(msg);
UBA.Common.LogHelperNet.Info(msg, null);
start = DateTime.Now;
//// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
stream = jobClient.GetJobOutput(jobResults.JobId);
}
}
catch (Exception ex)
{
Console.WriteLine("提交job失败：" + ex.Message);
UBA.Common.LogHelperNet.Error("提交job失败：", ex);
return "error";
}
msg = string.Format("处理完成job,耗时{0}秒", DateTime.Now.Subtract(start).TotalMilliseconds / 1000);
Console.WriteLine(msg);
UBA.Common.LogHelperNet.Info(msg, null);
return submitCallback(stream);
}
public static void LoadTweets(JobSubmissionCertificateCredential creds, AzureSettings settings)
{
var hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "Load tweets to external table",
StatusFolder = "/AAALoadTweets",
Query = "select identifiers.identifier, Z.X.estfrequency as tweetCount from (select explode(word_map) as X from ( SELECT context_ngrams(sentences(lower(tweetText)), array(null), 1000) as word_map FROM tweets ) struct) Z join identifiers on identifiers.identifier = Z.X.ngram[0]"
};
// Submit the Hive job
var jobClient = JobSubmissionClientFactory.Connect(creds);
var jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
WaitForJobCompletion(jobResults, jobClient);
// Print the Hive job output
var stream = jobClient.GetJobOutput(jobResults.JobId);
var reader = new StreamReader(stream);
Console.WriteLine(reader.ReadToEnd());
}
public void ServiceHost_JobSubmissionRecieved(object sender, JobSubmissionMessage e)
{
Trace.WriteLine("JobSubmissionRecieved Recieved User Id : " + e.idUsuario, "Warning");
try
{
// Obtener la cuenta de almacenamiento
// Para actualizar metadatos
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(VariablesConfiguracion.containerName);
// Obtencion de variables para la conxion con el cluster
string subscriptionID = VariablesConfiguracion.subscriptionID;
string certFriendlyName = VariablesConfiguracion.certFriendlyName;
string clusterName = VariablesConfiguracion.clusterName;
// Definicion de la tarea MapReduce
MapReduceJobCreateParameters mrJobDefinition = new MapReduceJobCreateParameters()
{
JarFile = "wasb:///CienciaCelularMR.jar",
ClassName = "Main",
StatusFolder = "wasb:///scicluster/test/status-" + e.idUsuario + "." + e.subIdUsuario,
};
mrJobDefinition.Arguments.Add("wasb:///" + e.nomEntrada);
mrJobDefinition.Arguments.Add("wasb:///scicluster/test/output-" + e.idUsuario + "." + e.subIdUsuario);
// Obtener el objeto certificate
X509Store store = new X509Store();
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = FindCertificate(StoreLocation.CurrentUser, StoreName.My, X509FindType.FindByThumbprint, VariablesConfiguracion.thumbprint);
JobSubmissionCertificateCredential creds = new JobSubmissionCertificateCredential(new Guid(subscriptionID), cert, clusterName);
// Se crea un cliente Hadoop para conectarse con HDInsight
var jobClient = JobSubmissionClientFactory.Connect(creds);
//Actualizacion de metadatos
CloudBlockBlob infoSimulation = container.GetBlockBlobReference(VariablesConfiguracion.infoSimulation + "-" + e.idUsuario);
infoSimulation.UploadText(VariablesConfiguracion.JOB_STARTING);
// Se lanza la ejecucion de los jobs MapReduce
JobCreationResults mrJobResults = jobClient.CreateMapReduceJob(mrJobDefinition);
// Esperar hasta que finalice la ejecucion
WaitForJobCompletion(mrJobResults, jobClient, e.idUsuario, e.subIdUsuario);
}
catch (Exception ex)
{
Trace.TraceError(ex.Message);
throw;
}
}
