Esempio n. 1
0
        static void Main(string[] args)
        {
            try
            {
                RunUsingStreamingUnit();

                IHadoop hadoop = Hadoop.Connect(MyConstants.AzureCluster,
                                                MyConstants.AzureUserName,
                                                MyConstants.HadoopUserName,
                                                MyConstants.AzurePassword,
                                                MyConstants.AzureStorageAccount,
                                                MyConstants.AzureStorageKey,
                                                MyConstants.AzureStorageContainer,
                                                false);

                var result = hadoop.MapReduceJob.ExecuteJob <MovieLensJob>();

                Console.WriteLine();
                Console.WriteLine("Job Run Information");
                Console.WriteLine();
                Console.WriteLine("Job Id: {0}", result.Id);
                Console.WriteLine("Exit Code: {0}", result.Info.ExitCode);
                Console.WriteLine("Standard Out");
                Console.WriteLine(result.Info.StandardOut);
                Console.WriteLine();
                Console.WriteLine("Standard Err");
                Console.WriteLine(result.Info.StandardError);
                Console.ReadLine();
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex);
                Console.ReadLine();
            }
        }
Esempio n. 2
0
        public void Run()
        {
            IHadoop hadoop = Hadoop.Connect();

            hadoop.MapReduceJob.ExecuteJob <HelloWorldJob>();
            Console.Read();
        }
Esempio n. 3
0
        static void Main(string[] args)
        {
            //establish job configuration
            var myConfig = new HadoopJobConfiguration {
                InputPath = "/demo/simple/in", OutputFolder = "/demo/simple/out"
            };

            //connect to cluster
            var          myUri     = new Uri("http://localhost");
            const string userName  = "******";
            string       passWord  = null;
            IHadoop      myCluster = Hadoop.Connect(myUri, userName, passWord);

            //execute mapreduce job
            MapReduceResult jobResult = myCluster.MapReduceJob.Execute <MySimpleMapper, MySimpleReducer>(myConfig);

            //write job result to console
            int exitCode = jobResult.Info.ExitCode;

            string exitStatus = exitCode == 0 ? "Success" : "Failure";

            exitStatus = exitCode + " (" + exitStatus + ")";

            Console.WriteLine();
            Console.Write("Exit Code = " + exitStatus);
            Console.Read();
        }
Esempio n. 4
0
 //Run Custom Map Reduce
 public static void DoCustomMapReduce()
 {
     Console.WriteLine("Starting MapReduce job. Remote login to your Name Node and check progress from JobTracker portal with the returned JobID...");
     IHadoop hadoop = Hadoop.Connect(Constants.azureClusterUri, Constants.clusterUser,
                                     Constants.hadoopUser, Constants.clusterPassword, Constants.storageAccount,
                                     Constants.storageAccountKey, Constants.container, true);
     //IHadoop hadoop = Hadoop.Connect();
     var output = hadoop.MapReduceJob.ExecuteJob <SquareRootJob>();
 }
        public void Run()
        {
            Uri uri = new Uri(@"https://10.0.0.4");
            //IHadoop hdp = Hadoop.Connect(uri,"cloudera","cloudera");
            IHadoop hdp = Hadoop.Connect();

            hdp.MapReduceJob.ExecuteJob <WCJobConf>();
            Console.Read();
        }
Esempio n. 6
0
        public static void DoCustomMapReduce()
        {
            Console.WriteLine("Starting MapReduce job...");
            IHadoop hadoop = Hadoop.Connect(Constants.azureClusterUri, Constants.clusterUser, Constants.hadoopUser,
                                            Constants.clusterPassword, Constants.storageAccount, Constants.storageAccountKey,
                                            Constants.container, true);

            var output = hadoop.MapReduceJob.ExecuteJob <RootJob>();
        }
Esempio n. 7
0
        static void Main(string[] args)
        {
            /* To create these locations on your own drive, open the Hadoop
             * console and then type in the following commands:
             *      hadoop fs -mkdir /user/OpenData
             *      hadoop fs -mkdir /user/OpenData/Police
             *      hadoop fs -copyFromLocal C:\Temp\Datasets\Police.csv /user/OpenData/Police/
             *      hadoop fs -mkdir /user/OpenData/Output
             */
            HadoopJobConfiguration config = new HadoopJobConfiguration();

            config.InputPath    = "/user/OpenData/Police";
            config.OutputFolder = "/user/OpenData/Output";

            //Replace the URI with your local machine name.
            //Note that password is ignored for the HDInsight emulator, so that can be whatever you want.
            Uri     clusterURI = new Uri("http://yourmachine");
            string  username   = "******";
            string  password   = null;
            IHadoop cluster    = Hadoop.Connect(clusterURI, username, password);

            Console.WriteLine("Crime Counter.  Select an option to continue:");
            Console.WriteLine("1) Raw count by crime");
            Console.WriteLine("2) Count by coordinates (4 spots after decimal)");

            var input = Console.ReadLine();

            MapReduceResult jobResult;

            switch (input)
            {
            case "1":
                jobResult = cluster.MapReduceJob.Execute <CrimeCount, TotalCrimeCount>(config);
                break;

            case "2":
                //Quick note:  if we just wanted to spit out all areas regardless of
                //number of crimes, we could just use the TotalCrimeCount class
                //and would not need to generate a new Reduce class.
                jobResult = cluster.MapReduceJob.Execute <CrimeLocation, TopCrimeLocations>(config);
                break;

            default:
                return;
            }

            int    exitcode   = jobResult.Info.ExitCode;
            string exitstatus = exitcode == 0 ? "Success" : "Failure";

            Console.WriteLine();
            Console.WriteLine("Exit Code = " + exitstatus);
            Console.Read();
        }
Esempio n. 8
0
        public static void DoCustomMapReduce()
        {
            //The credentials entered below are dummy values. Please input valid credentials and submit jobs
            Environment.SetEnvironmentVariable("HADOOP_HOME", @"C:\Syncfusion\BigData\3.2.0.20\BigDataSDK\\SDK\Hadoop");
            Environment.SetEnvironmentVariable("JAVA_HOME", @"C:\Syncfusion\BigData\3.2.0.20\BigDataSDK\\Java\jdk1.7.0_51");
            //Pass the cluster name
            string clusterName     = "https://{clustername}.azurehdinsight.net:";
            Uri    azureCluster    = new Uri(clusterName);
            string clusterUserName = "******"; // default - admin
            string clusterPassword = "******";

            //// This is the name of the account under which Hadoop will execute jobs.
            //// Normally this is just "Hadoop".
            string hadoopUserName = "******";

            //// Azure Storage Information.
            string azureStorageAccount   = "{storagename}.blob.core.windows.net";
            string azureStorageKey       = "{storagekey}";
            string azureStorageContainer = "{storagecontainer}";



            //Console.WriteLine("Starting MapReduce job. Remote login to your Name Node and check progress from JobTracker portal with the returned JobID...");

            IHadoop hadoop = Hadoop.Connect(azureCluster, clusterUserName,
                                            hadoopUserName, clusterPassword, azureStorageAccount,
                                            azureStorageKey, azureStorageContainer, true);
            // Create or overwrite the "myblob" blob with contents from a local file.
            var fileStream = File.ReadAllText(@"..//..//data/NASA_Access_Log");

            hadoop.StorageSystem.WriteAllText(FindReplace._input1HDFS, fileStream);
            Console.WriteLine("Input file uploaded.......\n\n");
            Console.WriteLine("Find and Replace Operation.\n\nImplementation of Find and Replace operations in native MapReduce through C#");
            Console.WriteLine("Execution begins......\n");

            //connect to HDInsightcluster
            MapReduceResult result = hadoop.MapReduceJob.ExecuteJob <FindReplace>();

            Console.WriteLine();
            Console.WriteLine("Job Run Information");
            Console.WriteLine();
            Console.WriteLine("Job Id: {0}", result.Id);
            Console.WriteLine("Exit Code: {0}", result.Info.ExitCode);
            Console.WriteLine("Standard Out");
            Console.WriteLine(result.Info.StandardOut);
            Console.WriteLine();
            Console.WriteLine("Standard Err");
            Console.WriteLine(result.Info.StandardError);
            Console.ReadKey();
        }
Esempio n. 9
0
        private static void DoMapReduce()
        {
            try
            {
                SetEnvironment();
                UserInteraction();
                var config = GetHadoopConfiguration();

                IHadoop myCluster = null;
                if (IsRemote)
                {
                    if (!CreateConfigurationDirectory())
                    {
                        Console.WriteLine("Failed to create configuration directory");
                    }
                    else
                    {
                        if (IsSecured)
                        {
                            myCluster = ConnectToSecuredRemoteCluster();
                        }
                        else
                        {
                            myCluster = ConnectToRemoteCluster();
                        }
                    }
                }
                else
                {
                    myCluster = ConnectToLocalPseudoCluster();
                }

                if (myCluster != null)
                {
                    //execute mapreduce job
                    Console.WriteLine("\n\nExecution begins......\n");

                    //passing the Mapper and Reducer
                    var jobResult = myCluster.MapReduceJob.Execute <SimpleMapper, SimpleReducer>(config);
                    UpdateExecutionStatus(jobResult.Info.ExitCode);
                }
                UpdateExecutionStatus(1);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                Console.Read();
            }
        }
        public HDInsightRepository()
        {
            Environment.SetEnvironmentVariable("HADOOP_HOME", @"c:\hadoop");
            Environment.SetEnvironmentVariable("Java_HOME", @"c:\hadoop\jvm");

            hadoop = Hadoop.Connect(
                new Uri("<clusterurl>"),
                "<username>",
                "<hadoopuser>",
                "<pass>",
                "<storageaccount>",
                "<storagekey>",
                "default",
                true
                );
        }
Esempio n. 11
0
        static void Main(string[] args)
        {
            HadoopJobConfiguration myConfig = new HadoopJobConfiguration();

            myConfig.InputPath    = "/world/in/worldbank";
            myConfig.OutputFolder = "/world/out";



            Uri myUri = new Uri("http://*****:*****@"c:\hadoop");
            Environment.SetEnvironmentVariable("Java_HOME", @"c:\hadoop\jvm");

            IHadoop myCluster = Microsoft.Hadoop.MapReduce.Hadoop.Connect(myUri, userName, passWord);

            MapReduceResult jobResult = myCluster.MapReduceJob.Execute <ExtractValuesForIndicatorsMapper, IndicatorsReducer>(myConfig);

            HadoopJobConfiguration myConfig2 = new HadoopJobConfiguration();

            myConfig2.InputPath    = "/world/out";
            myConfig2.OutputFolder = "/world/out2";
            //MapReduceResult jobResult2 = myCluster.MapReduceJob.Execute<GroupValuesMapper, GroupValuesReducer>(myConfig2);



            int exitCode = jobResult.Info.ExitCode;



            string exitStatus = "Failure";

            if (exitCode == 0)
            {
                exitStatus = "Success";
            }

            exitStatus = exitCode + " (" + exitStatus + ")";

            Console.WriteLine();

            Console.Write("Exit Code = " + exitStatus);
        }
Esempio n. 12
0
        private static void Main(string[] args)
        {
            IHadoop hadoop = Hadoop.Connect(MyConstants.AzureCluster,
                                            MyConstants.AzureUserName,
                                            MyConstants.HadoopUserName,
                                            MyConstants.AzurePassword,
                                            MyConstants.AzureStorageAccount,
                                            MyConstants.AzureStorageKey,
                                            MyConstants.AzureStorageContainer,
                                            false);

            //var result = hadoop.MapReduceJob.ExecuteJob<AggregatePurchaseJob>();
            //Console.WriteLine("Aggregate Purchase Job complete: {0}", result.Id);
            var chainedResult = hadoop.MapReduceJob.ExecuteJob <NewUserPurchasesByContinentJob>();

            Console.WriteLine("New User Purchases By Continent job complete: {0}", chainedResult.Id);

            Console.ReadLine();
        }