private string GetMapData() { try { HadoopJobConfiguration myConfig = new HadoopJobConfiguration(); myConfig.InputPath = "/demo/simple/in"; myConfig.OutputFolder = "/demo/simple/out"; //connect to cluster Uri myUri = new Uri("hdfs://0.0.0.0:19000"); string userName = null; string passWord = null; //Microsoft.Hadoop.WebHDFS.WebHDFSClient.MapReduce.IHadoop myCluster = Microsoft.Hadoop.MapReduce.Hadoop.Connect(myUri, userName, passWord); Microsoft.Hadoop.WebHDFS.WebHDFSClient client = new Microsoft.Hadoop.WebHDFS.WebHDFSClient(myUri, userName); return(client.GetHomeDirectory().Result); } catch (Exception ex) { throw ex; } return(string.Empty); }
public static void Main(string[] args) { //establish job configuration HadoopJobConfiguration myConfig = new HadoopJobConfiguration(); myConfig.InputPath = "/demo/simple/in"; myConfig.OutputFolder = "/demo/simple/out"; //connect to cluster Uri myUri = new Uri("http://WSB-201:10001"); string userName = "******"; string passWord = null; IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord); //execute mapreduce job var start = DateTime.Now; MapReduceResult jobResult = myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myConfig); var result = DateTime.Now - start; Console.WriteLine($"{result.TotalMilliseconds} ms"); //write job result to console int exitCode = jobResult.Info.ExitCode; string exitStatus = "Failure"; if (exitCode == 0) exitStatus = "Success"; exitStatus = exitCode + " (" +exitStatus + ")"; Console.WriteLine(); Console.Write("Exit Code = " +exitStatus); Console.Read(); }
private string GetMapData() { try { HadoopJobConfiguration myConfig = new HadoopJobConfiguration(); myConfig.InputPath = "/demo/simple/in"; myConfig.OutputFolder = "/demo/simple/out"; //connect to cluster Uri myUri = new Uri("hdfs://0.0.0.0:19000"); string userName = null; string passWord = null; //Microsoft.Hadoop.WebHDFS.WebHDFSClient.MapReduce.IHadoop myCluster = Microsoft.Hadoop.MapReduce.Hadoop.Connect(myUri, userName, passWord); Microsoft.Hadoop.WebHDFS.WebHDFSClient client = new Microsoft.Hadoop.WebHDFS.WebHDFSClient(myUri, userName); return client.GetHomeDirectory().Result; } catch (Exception ex) { throw ex; } return string.Empty; }
static void Main(string[] args) { //establish job configuration var myConfig = new HadoopJobConfiguration { InputPath = "/demo/simple/in", OutputFolder = "/demo/simple/out" }; //connect to cluster var myUri = new Uri("http://localhost"); const string userName = "******"; string passWord = null; IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord); //execute mapreduce job MapReduceResult jobResult = myCluster.MapReduceJob.Execute <MySimpleMapper, MySimpleReducer>(myConfig); //write job result to console int exitCode = jobResult.Info.ExitCode; string exitStatus = exitCode == 0 ? "Success" : "Failure"; exitStatus = exitCode + " (" + exitStatus + ")"; Console.WriteLine(); Console.Write("Exit Code = " + exitStatus); Console.Read(); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { HadoopJobConfiguration config = new HadoopJobConfiguration(); config.InputPath = "Deliveries-tsv"; config.OutputFolder = "output/kmPerVehicle"; return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { HadoopJobConfiguration config = new HadoopJobConfiguration(); config.InputPath = "Input/sqrt"; config.OutputFolder = "Output/sqrt"; return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { var config = new HadoopJobConfiguration(); config.InputPath = "input/rawdata"; config.OutputFolder = "output/data"; return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { HadoopJobConfiguration config = new HadoopJobConfiguration(); config.InputPath = "asv://[email protected]/tweet/input/furious7.txt"; config.OutputFolder = "asv://[email protected]/tweet/output"; return(config); }
public MapReduceResult Work(HadoopJobConfiguration config, out StatusResult statusCode) { var myCluster = ClusterFactory.GetCluster(); var jobResult = myCluster.MapReduceJob.Execute <TMapper, TReducer>(config); statusCode = jobResult.Info.ExitCode.ToStatusResult(); return(jobResult); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { var config = new HadoopJobConfiguration(); // config.DeleteOutputFolder = true; config.InputPath = "/input/"; config.OutputFolder = "/output/"; return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { HadoopJobConfiguration config = new HadoopJobConfiguration(); config.InputPath = "input/SqrtJob"; config.OutputFolder = "output/SqrtJob"; //config return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { var config = new HadoopJobConfiguration { InputPath = "wasb:///test2.txt", OutputFolder = "wasb:///myresclust" }; return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { var config = new HadoopJobConfiguration { InputPath = Constants.wasbPath + "/example/data/Numbers.txt", OutputFolder = Constants.wasbPath + "/example/data/SqaureRootOutput" }; return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { HadoopJobConfiguration config = new HadoopJobConfiguration(); config.Verbose = true; config.InputPath = _input1HDFS; config.OutputFolder = s_outputFolderHDFS; config.AdditionalGenericArguments.Add("-D \"mapred.map.tasks=3\""); // example of controlling arbitrary hadoop options. return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { var configuration = new HadoopJobConfiguration { InputPath = MyConstants.AzureStorageVaultRoot + "/handsonlabs/lab1/output/part-00000", OutputFolder = MyConstants.AzureStorageVaultRoot + "/handsonlabs/lab2/output" }; return(configuration); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { var jobConf = new HadoopJobConfiguration() { InputPath = "/in", OutputFolder = "/out", }; return(jobConf); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { var configuration = new HadoopJobConfiguration { InputPath = MyConstants.AzureStorageVaultRoot + "/handsonlabs/lab1Results.log", OutputFolder = MyConstants.AzureStorageVaultRoot + "/handsonlabs/output/newUserPurchasesByContinent/" + Guid.NewGuid().ToString("N") }; return(configuration); }
private static HadoopJobConfiguration GetHadoopConfiguration() { HadoopJobConfiguration config = new HadoopJobConfiguration(); config.InputPath = "/Data/NASA_Access_Log"; config.OutputFolder = "/output"; Console.WriteLine("\n\n\nInput Path :" + config.InputPath); Console.WriteLine("\nOutput Folder :" + config.OutputFolder); return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { var config = new HadoopJobConfiguration { InputPath = "wasb://[email protected]/input", OutputFolder = "wasb://[email protected]/output", DeleteOutputFolder = false }; return(config); }
public override HadoopJobConfiguration Configure(ExecutorContext context) { var configuration = new HadoopJobConfiguration { InputPath = MyConstants.AzureStorageVaultRoot + "/handsonlabs/data/ua.base", OutputFolder = string.Format("{0}{1}", MyConstants.AzureStorageVaultRoot, "/handsonlabs/lab5/output") }; return(configuration); }
static void Main(string[] args) { HadoopJobConfiguration conf = new HadoopJobConfiguration() { InputPath = "/demo/in", OutputFolder = "/demo/out" }; Hadoop.Connect(new Uri("http://win8-dev-pc/"), "hadoop", "") .MapReduceJob .Execute<Mapper, Reducer>(conf); }
static void Main(string[] args) { /* To create these locations on your own drive, open the Hadoop * console and then type in the following commands: * hadoop fs -mkdir /user/OpenData * hadoop fs -mkdir /user/OpenData/Police * hadoop fs -copyFromLocal C:\Temp\Datasets\Police.csv /user/OpenData/Police/ * hadoop fs -mkdir /user/OpenData/Output */ HadoopJobConfiguration config = new HadoopJobConfiguration(); config.InputPath = "/user/OpenData/Police"; config.OutputFolder = "/user/OpenData/Output"; //Replace the URI with your local machine name. //Note that password is ignored for the HDInsight emulator, so that can be whatever you want. Uri clusterURI = new Uri("http://yourmachine"); string username = "******"; string password = null; IHadoop cluster = Hadoop.Connect(clusterURI, username, password); Console.WriteLine("Crime Counter. Select an option to continue:"); Console.WriteLine("1) Raw count by crime"); Console.WriteLine("2) Count by coordinates (4 spots after decimal)"); var input = Console.ReadLine(); MapReduceResult jobResult; switch (input) { case "1": jobResult = cluster.MapReduceJob.Execute <CrimeCount, TotalCrimeCount>(config); break; case "2": //Quick note: if we just wanted to spit out all areas regardless of //number of crimes, we could just use the TotalCrimeCount class //and would not need to generate a new Reduce class. jobResult = cluster.MapReduceJob.Execute <CrimeLocation, TopCrimeLocations>(config); break; default: return; } int exitcode = jobResult.Info.ExitCode; string exitstatus = exitcode == 0 ? "Success" : "Failure"; Console.WriteLine(); Console.WriteLine("Exit Code = " + exitstatus); Console.Read(); }
static void Main(string[] args) { HadoopJobConfiguration conf = new HadoopJobConfiguration() { InputPath = "/demo/in", OutputFolder = "/demo/out" }; Hadoop.Connect(new Uri("http://win8-dev-pc/"), "hadoop", "") .MapReduceJob .Execute <Mapper, Reducer>(conf); }
static void Main(string[] args) { /* To create these locations on your own drive, open the Hadoop console and then type in the following commands: hadoop fs -mkdir /user/OpenData hadoop fs -mkdir /user/OpenData/Police hadoop fs -copyFromLocal C:\Temp\Datasets\Police.csv /user/OpenData/Police/ hadoop fs -mkdir /user/OpenData/Output */ HadoopJobConfiguration config = new HadoopJobConfiguration(); config.InputPath = "/user/OpenData/Police"; config.OutputFolder = "/user/OpenData/Output"; //Replace the URI with your local machine name. //Note that password is ignored for the HDInsight emulator, so that can be whatever you want. Uri clusterURI = new Uri("http://yourmachine"); string username = "******"; string password = null; IHadoop cluster = Hadoop.Connect(clusterURI, username, password); Console.WriteLine("Crime Counter. Select an option to continue:"); Console.WriteLine("1) Raw count by crime"); Console.WriteLine("2) Count by coordinates (4 spots after decimal)"); var input = Console.ReadLine(); MapReduceResult jobResult; switch (input) { case "1": jobResult = cluster.MapReduceJob.Execute<CrimeCount, TotalCrimeCount>(config); break; case "2": //Quick note: if we just wanted to spit out all areas regardless of //number of crimes, we could just use the TotalCrimeCount class //and would not need to generate a new Reduce class. jobResult = cluster.MapReduceJob.Execute<CrimeLocation, TopCrimeLocations>(config); break; default: return; } int exitcode = jobResult.Info.ExitCode; string exitstatus = exitcode == 0 ? "Success" : "Failure"; Console.WriteLine(); Console.WriteLine("Exit Code = " + exitstatus); Console.Read(); }
static void Main(string[] args) { HadoopJobConfiguration myConfig = new HadoopJobConfiguration(); myConfig.InputPath = "/world/in/worldbank"; myConfig.OutputFolder = "/world/out"; Uri myUri = new Uri("http://*****:*****@"c:\hadoop"); Environment.SetEnvironmentVariable("Java_HOME", @"c:\hadoop\jvm"); IHadoop myCluster = Microsoft.Hadoop.MapReduce.Hadoop.Connect(myUri, userName, passWord); MapReduceResult jobResult = myCluster.MapReduceJob.Execute<ExtractValuesForIndicatorsMapper, IndicatorsReducer>(myConfig); HadoopJobConfiguration myConfig2 = new HadoopJobConfiguration(); myConfig2.InputPath = "/world/out"; myConfig2.OutputFolder = "/world/out2"; //MapReduceResult jobResult2 = myCluster.MapReduceJob.Execute<GroupValuesMapper, GroupValuesReducer>(myConfig2); int exitCode = jobResult.Info.ExitCode; string exitStatus = "Failure"; if (exitCode == 0) exitStatus = "Success"; exitStatus = exitCode + " (" + exitStatus + ")"; Console.WriteLine(); Console.Write("Exit Code = " + exitStatus); }
static void Main(string[] args) { HadoopJobConfiguration myConfig = new HadoopJobConfiguration(); myConfig.InputPath = "/world/in/worldbank"; myConfig.OutputFolder = "/world/out"; Uri myUri = new Uri("http://*****:*****@"c:\hadoop"); Environment.SetEnvironmentVariable("Java_HOME", @"c:\hadoop\jvm"); IHadoop myCluster = Microsoft.Hadoop.MapReduce.Hadoop.Connect(myUri, userName, passWord); MapReduceResult jobResult = myCluster.MapReduceJob.Execute <ExtractValuesForIndicatorsMapper, IndicatorsReducer>(myConfig); HadoopJobConfiguration myConfig2 = new HadoopJobConfiguration(); myConfig2.InputPath = "/world/out"; myConfig2.OutputFolder = "/world/out2"; //MapReduceResult jobResult2 = myCluster.MapReduceJob.Execute<GroupValuesMapper, GroupValuesReducer>(myConfig2); int exitCode = jobResult.Info.ExitCode; string exitStatus = "Failure"; if (exitCode == 0) { exitStatus = "Success"; } exitStatus = exitCode + " (" + exitStatus + ")"; Console.WriteLine(); Console.Write("Exit Code = " + exitStatus); }
static void Main(string[] args) { HadoopJobConfiguration myconfig = new HadoopJobConfiguration(); myconfig.InputPath = "/in"; myconfig.OutputFolder = "/out"; //connect to cluster Uri myUri = new Uri("http://127.0.0.1:50070"); Copy(myUri, "/input"); string userName = "******"; string passWord = null; IHadoop myCluster = Microsoft.Hadoop.MapReduce.Hadoop.Connect(myUri, userName, null); //execute mapreduce job MapReduceResult jobResult = myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myconfig); int exitCode = jobResult.Info.ExitCode; string exitStatus = "Failure"; if (exitCode == 0) exitStatus = "Success"; exitStatus = exitCode + " (" + exitStatus + ")"; Console.WriteLine(); Console.Write("Exit Code = " + exitStatus); }
// Main Method static void Main(string[] args) { // Load Data in to HDInsight HadoopJobConfiguration practice_jobConfig = new HadoopJobConfiguration() { InputPath = "/user/Jayakaran/Input/NHS/Practices", OutputFolder = "/user/Jayakaran/Output/NHS/Practices", DeleteOutputFolder = true }; HadoopJobConfiguration prescription_jobConfig = new HadoopJobConfiguration() { InputPath = "/user/Jayakaran/Input/NHS/Prescription", OutputFolder = "/user/Jayakaran/Output/NHS/Prescription", DeleteOutputFolder = true }; HadoopJobConfiguration combined_jobConfig = new HadoopJobConfiguration() { InputPath = "/user/Jayakaran/Input/NHS/Combined", OutputFolder = "/user/Jayakaran/Output/NHS/Combined", DeleteOutputFolder = true }; // Call Jobs // Question 1 How many practices are in London? Hadoop.Connect().MapReduceJob.Execute<PracticesCountDataMapper, PracticesCountDataReducer>(practice_jobConfig); // Question 2 What was the average actual cost of all peppermint oil prescriptions? Hadoop.Connect().MapReduceJob.Execute<PrescriptionFilterDataMapper, PrescriptionFilterDataReducer>(prescription_jobConfig); // Question 3 Which 5 post codes have the highest actual spend, and how much did each spend in total? Hadoop.Connect().MapReduceJob.Execute<CombinedDataMapper, CombinedDataReducer>(combined_jobConfig); System.Console.Read(); //using to catch console }
static void Main(string[] args) { //establish job configuration var myConfig = new HadoopJobConfiguration {InputPath = "/demo/simple/in", OutputFolder = "/demo/simple/out"}; //connect to cluster var myUri = new Uri("http://localhost"); const string userName = "******"; string passWord = null; IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord); //execute mapreduce job MapReduceResult jobResult = myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myConfig); //write job result to console int exitCode = jobResult.Info.ExitCode; string exitStatus = exitCode == 0 ? "Success" : "Failure"; exitStatus = exitCode + " (" + exitStatus + ")"; Console.WriteLine(); Console.Write("Exit Code = " + exitStatus); Console.Read(); }
// Main Method static void Main(string[] args) { // Load Data in to HDInsight HadoopJobConfiguration practice_jobConfig = new HadoopJobConfiguration() { InputPath = "/user/Jayakaran/Input/NHS/Practices", OutputFolder = "/user/Jayakaran/Output/NHS/Practices", DeleteOutputFolder = true }; HadoopJobConfiguration prescription_jobConfig = new HadoopJobConfiguration() { InputPath = "/user/Jayakaran/Input/NHS/Prescription", OutputFolder = "/user/Jayakaran/Output/NHS/Prescription", DeleteOutputFolder = true }; HadoopJobConfiguration combined_jobConfig = new HadoopJobConfiguration() { InputPath = "/user/Jayakaran/Input/NHS/Combined", OutputFolder = "/user/Jayakaran/Output/NHS/Combined", DeleteOutputFolder = true }; // Call Jobs // Question 1 How many practices are in London? Hadoop.Connect().MapReduceJob.Execute <PracticesCountDataMapper, PracticesCountDataReducer>(practice_jobConfig); // Question 2 What was the average actual cost of all peppermint oil prescriptions? Hadoop.Connect().MapReduceJob.Execute <PrescriptionFilterDataMapper, PrescriptionFilterDataReducer>(prescription_jobConfig); // Question 3 Which 5 post codes have the highest actual spend, and how much did each spend in total? Hadoop.Connect().MapReduceJob.Execute <CombinedDataMapper, CombinedDataReducer>(combined_jobConfig); System.Console.Read(); //using to catch console }
private static HadoopJobConfiguration Required(this HadoopJobConfiguration config) { config.FilesToInclude.Add("Microsoft.WindowsAzure.Management.Framework.Threading.dll"); return(config); }