Exemplo n.º 1
0
        /// <summary>
        /// Runs the sample (including pre- and post-cleaning the cluster)
        /// </summary>
        private void RunSample()
        {
            //Connects to cluster based upon the information from config file
            var cluster = new Cluster();
            this.ExecuteStep("Connecting to cluster", () => cluster.Connect(this.certificate, this.subscription, this.clusterName, this.storageAccountName, this.storageAccountKey));
            
            //Cleans cluster
            this.ExecuteStep("Cleaning cluster", cluster.CleanUp);
            
            //Read input CSV files, serialize them and upload to Azure blob
            this.ExecuteStep(
                "Uploading files",
                () =>
                {
                    //The sample expects an excerpt from historical AMEX stock data as distributed by Infochimps, http://www.infochimps.com/
                    //CSV files are distributed with the sample
                    for (char i = 'A'; i < 'C'; i++)
                    {
                        var input = Path.Combine(this.dataSetLocation, "AMEX_daily_prices_" + i + ".csv");

                        // Convert CSV file to Avro format using Reflection and auto generated C# types code,
                        // write it to stream and upload it to cluster
                        using (var stream = new MemoryStream())
                        {
                            SerializeCsv(input, stream);
                            cluster.UploadAvro("AMEX_daily_prices_" + i + ".avro", stream);
                        }
                    }
                });

            //Create Hive table
            this.ExecuteStep("Creating Hive table", cluster.CreateStocksTable);

            //Execute a Hive query
            this.ExecuteStep(
                "Executing Query ",
                () =>
                {
                    const string Query = "SELECT YEAR(stockdate), AVG(closeprice) FROM Stocks WHERE symbol='AIP' GROUP BY year(stockdate);";
                    Console.WriteLine(Query);
                    Console.Out.WriteLine(
                        cluster.Query(new HiveJobCreateParameters { Query = Query, JobName = "YearlyAverages" }));
                },
                true);
            
            //Another Hive query
            //Commented out to diminsh the sample run time
            //this.ExecuteStep(
            //   "Executing Query ",
            //    () =>
            //    {
            //        const string Query = "SELECT YEAR(stockdate), MAX(closeprice) FROM Stocks GROUP BY year(stockdate);";
            //        Console.WriteLine(Query);
            //        Console.Out.WriteLine(
            //            cluster.Query(new HiveJobCreateParameters { Query = Query, JobName = "YearlyMaxes" }));
            //    },
            //    true);
            
            //Clean the cluster
            this.ExecuteStep("Cleaning cluster", cluster.CleanUp);
        }
Exemplo n.º 2
0
 /// <summary>
 /// Cleans the cluster.
 /// </summary>
 private void CleanCluster()
 {
     //Connects to cluster based upon the information from config file
     var cluster = new Cluster();
     this.ExecuteStep("Connecting to cluster", () => cluster.Connect(this.certificate, this.subscription, this.clusterName, this.storageAccountName, this.storageAccountKey));
     
     //Call the actual clean up procedure
     this.ExecuteStep("Cleaning cluster", cluster.CleanUp);
 }