/// <summary>
/// Deletes a directory in Azure Blob
/// </summary>
/// <param name="directory">The directory.</param>
private void DeleteDirectory(CloudBlobDirectory directory)
{
try
{
foreach (var item in directory.ListBlobs())
{
var blockBlob = item as CloudBlockBlob;
if (blockBlob != null)
{
blockBlob.DeleteIfExists();
}
else
{
var dir = item as CloudBlobDirectory;
if (dir != null)
{
this.DeleteDirectory(dir);
}
}
}
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while cleaning the cluster\n" + e);
}
}
/// <summary>
/// Uploads the specified path.
/// </summary>
/// <param name="path">The path.</param>
/// <param name="stream">The stream.</param>
private void Upload(string path, Stream stream)
{
CloudBlockBlob blockBlob = null;
try
{
// Create a block Blob for the file
blockBlob = this.container.GetBlockBlobReference(RootDirectory + "/" + path);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while getting a block reference from the cluster\n" + e);
}
// Rewind the stream
stream.Seek(0, SeekOrigin.Begin);
try
{
// Upload the stream content to designated block Blob
blockBlob.UploadFromStream(stream);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while uploading data to cluster\n" + e);
}
}
/// <summary>
/// Performs HQL query and returns the query results.
/// </summary>
/// <param name="jobParams">The query parameters.</param>
/// <returns>The query result.</returns>
public string Query(HiveJobCreateParameters jobParams)
{
// Assign status folder
jobParams.StatusFolder = RootDirectory + "/status";
JobCreationResults jobDetails = null;
try
{
// Create Hive job
jobDetails = this.job.CreateHiveJob(jobParams);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while creating a Hive job\n" + e);
}
JobDetails jobInProgress = null;
try
{
// Get job status
jobInProgress = this.job.GetJob(jobDetails.JobId);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while getting Hive job status\n" + e);
}
// If job is not finished then sleep until the next client polling interval
while (jobInProgress.StatusCode != JobStatusCode.Completed &&
jobInProgress.StatusCode != JobStatusCode.Failed)
{
try
{
// Get job status
jobInProgress = this.job.GetJob(jobDetails.JobId);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while getting Hive job status\n" + e);
}
Thread.Sleep(this.client.PollingInterval);
}
try
{
// Job is finished; get its output stream, read it, and return the value
return(new StreamReader(this.job.GetJobOutput(jobDetails.JobId)).ReadToEnd());
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while reading Hibe job result\n" + e);
}
return(string.Empty);
}
/// <summary>
/// Program main entry.
/// </summary>
/// <param name="args">The arguments.</param>
private static void Main(string[] args)
{
var sample = new AvroHdiSample();
//Read Azure subscription and HDIsnight cluster information from config file
sample.ParseAppConfig();
//Parse command like arguments
sample.ParseArguments(args);
//Execute the major part of the sample
sample.Execute();
}
/// <summary>
/// Program main entry.
/// </summary>
/// <param name="args">The arguments.</param>
private static void Main(string[] args)
{
var sample = new AvroHdiSample();
//Read Azure subscription and HDIsnight cluster information from config file
sample.ParseAppConfig();
//Parse command like arguments
sample.ParseArguments(args);
//Execute the major part of the sample
sample.Execute();
}
/// <summary>
/// Connects to HDInsight cluster.
/// </summary>
/// <param name="certificate">The certificate.</param>
/// <param name="subscription">The subscription.</param>
/// <param name="clusterName">Name of the cluster.</param>
/// <param name="storageAccountName">Name of the storage account.</param>
/// <param name="storageAccountKey">The storage account key.</param>
public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey)
{
// Obtain the certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0);
if (cert == null)
{
AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!");
}
// Connect to the cluster using the certificate and the subscription
try
{
this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert));
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e);
}
this.cluster = this.client.GetCluster(clusterName);
if (this.cluster == null)
{
AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName);
}
// Create a job client
this.job = JobSubmissionClientFactory.Connect(
new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName));
// Create an Azure storage client
// We will use this client to upload files to Azure storage account
// which is used by HDInsight cluster.
var storageAccount = CloudStorageAccount.Parse(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey);
var blobClient = storageAccount.CreateCloudBlobClient();
this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container);
}
