/// <summary>
/// Runs the sample (including pre- and post-cleaning the cluster)
/// </summary>
private void RunSample()
{
//Connects to cluster based upon the information from config file
var cluster = new Cluster();
this.ExecuteStep("Connecting to cluster", () => cluster.Connect(this.certificate, this.subscription, this.clusterName, this.storageAccountName, this.storageAccountKey));
//Cleans cluster
this.ExecuteStep("Cleaning cluster", cluster.CleanUp);
//Read input CSV files, serialize them and upload to Azure blob
this.ExecuteStep(
"Uploading files",
() =>
{
//The sample expects an excerpt from historical AMEX stock data as distributed by Infochimps, http://www.infochimps.com/
//CSV files are distributed with the sample
for (char i = 'A'; i < 'C'; i++)
{
var input = Path.Combine(this.dataSetLocation, "AMEX_daily_prices_" + i + ".csv");
// Convert CSV file to Avro format using Reflection and auto generated C# types code,
// write it to stream and upload it to cluster
using (var stream = new MemoryStream())
{
SerializeCsv(input, stream);
cluster.UploadAvro("AMEX_daily_prices_" + i + ".avro", stream);
}
}
});
//Create Hive table
this.ExecuteStep("Creating Hive table", cluster.CreateStocksTable);
//Execute a Hive query
this.ExecuteStep(
"Executing Query ",
() =>
{
const string Query = "SELECT YEAR(stockdate), AVG(closeprice) FROM Stocks WHERE symbol='AIP' GROUP BY year(stockdate);";
Console.WriteLine(Query);
Console.Out.WriteLine(
cluster.Query(new HiveJobCreateParameters { Query = Query, JobName = "YearlyAverages" }));
},
true);
//Another Hive query
//Commented out to diminsh the sample run time
//this.ExecuteStep(
// "Executing Query ",
// () =>
// {
// const string Query = "SELECT YEAR(stockdate), MAX(closeprice) FROM Stocks GROUP BY year(stockdate);";
// Console.WriteLine(Query);
// Console.Out.WriteLine(
// cluster.Query(new HiveJobCreateParameters { Query = Query, JobName = "YearlyMaxes" }));
// },
// true);
//Clean the cluster
this.ExecuteStep("Cleaning cluster", cluster.CleanUp);
}
/// <summary>
/// Cleans the cluster.
/// </summary>
private void CleanCluster()
{
//Connects to cluster based upon the information from config file
var cluster = new Cluster();
this.ExecuteStep("Connecting to cluster", () => cluster.Connect(this.certificate, this.subscription, this.clusterName, this.storageAccountName, this.storageAccountKey));
//Call the actual clean up procedure
this.ExecuteStep("Cleaning cluster", cluster.CleanUp);
}
