/// <summary> /// Retrieves a map of /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// s to the /// <see cref="Org.Apache.Hadoop.Mapred.Mapper{K1, V1, K2, V2}"/> /// class that /// should be used for them. /// </summary> /// <param name="conf">The confuration of the job</param> /// <seealso cref="AddInputPath(Org.Apache.Hadoop.Mapred.JobConf, Org.Apache.Hadoop.FS.Path, System.Type{T}, System.Type{T}) /// "/> /// <returns>A map of paths to mappers for the job</returns> internal static IDictionary <Path, Type> GetMapperTypeMap(JobConf conf) { if (conf.Get("mapreduce.input.multipleinputs.dir.mappers") == null) { return(Sharpen.Collections.EmptyMap()); } IDictionary <Path, Type> m = new Dictionary <Path, Type>(); string[] pathMappings = conf.Get("mapreduce.input.multipleinputs.dir.mappers").Split (","); foreach (string pathMapping in pathMappings) { string[] split = pathMapping.Split(";"); Type mapClass; try { mapClass = (Type)conf.GetClassByName(split[1]); } catch (TypeLoadException e) { throw new RuntimeException(e); } m[new Path(split[0])] = mapClass; } return(m); }
/// <seealso cref="Org.Apache.Hadoop.Mapreduce.Protocol.ClientProtocol.GetSystemDir() /// "/> public override string GetSystemDir() { Path sysDir = new Path(conf.Get(JTConfig.JtSystemDir, "/tmp/hadoop/mapred/system" )); return(fs.MakeQualified(sysDir).ToString()); }
/* * (non-Javadoc) * * @see * org.apache.hadoop.mapred.MapReduceBase#configure(org.apache.hadoop.mapred * .JobConf) */ public override void Configure(JobConf conf) { // MapReduceBase try { config = new ConfigExtractor(conf); ConfigExtractor.DumpOptions(config); filesystem = config.GetBaseDirectory().GetFileSystem(conf); } catch (Exception e) { Log.Error("Unable to setup slive " + StringUtils.StringifyException(e)); throw new RuntimeException("Unable to setup slive configuration", e); } if (conf.Get(MRJobConfig.TaskAttemptId) != null) { this.taskId = TaskAttemptID.ForName(conf.Get(MRJobConfig.TaskAttemptId)).GetTaskID ().GetId(); } else { // So that branch-1/0.20 can run this same code as well this.taskId = TaskAttemptID.ForName(conf.Get("mapred.task.id")).GetTaskID().GetId (); } }
// Mappers that simply checks if the desired user env are present or not public override void Configure(JobConf job) { bool oldConfigs = job.GetBoolean(OldConfigs, false); if (oldConfigs) { string javaOpts = job.Get(JobConf.MapredTaskJavaOpts); NUnit.Framework.Assert.IsNotNull(JobConf.MapredTaskJavaOpts + " is null!", javaOpts ); NUnit.Framework.Assert.AreEqual(JobConf.MapredTaskJavaOpts + " has value of: " + javaOpts, javaOpts, TaskOptsVal); } else { string mapJavaOpts = job.Get(JobConf.MapredMapTaskJavaOpts); NUnit.Framework.Assert.IsNotNull(JobConf.MapredMapTaskJavaOpts + " is null!", mapJavaOpts ); NUnit.Framework.Assert.AreEqual(JobConf.MapredMapTaskJavaOpts + " has value of: " + mapJavaOpts, mapJavaOpts, MapOptsVal); } string path = job.Get("path"); // check if the pwd is there in LD_LIBRARY_PATH string pwd = Runtime.Getenv("PWD"); NUnit.Framework.Assert.IsTrue("LD doesnt contain pwd", Runtime.Getenv("LD_LIBRARY_PATH" ).Contains(pwd)); // check if X=$X:/abc works for LD_LIBRARY_PATH CheckEnv("LD_LIBRARY_PATH", "/tmp", "append"); // check if X=y works for an already existing parameter CheckEnv("LANG", "en_us_8859_1", "noappend"); // check if X=/tmp for a new env variable CheckEnv("MY_PATH", "/tmp", "noappend"); // check if X=$X:/tmp works for a new env var and results into :/tmp CheckEnv("NEW_PATH", FilePath.pathSeparator + "/tmp", "noappend"); // check if X=$(tt's X var):/tmp for an old env variable inherited from // the tt if (Shell.Windows) { // On Windows, PATH is replaced one more time as part of default config // of "mapreduce.admin.user.env", i.e. on Windows, // "mapreduce.admin.user.env" is set to // "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin" string hadoopHome = Runtime.Getenv("HADOOP_COMMON_HOME"); if (hadoopHome == null) { hadoopHome = string.Empty; } string hadoopLibLocation = hadoopHome + "\\bin"; path += FilePath.pathSeparator + hadoopLibLocation; path += FilePath.pathSeparator + path; } CheckEnv("PATH", path + FilePath.pathSeparator + "/tmp", "noappend"); string jobLocalDir = job.Get(MRJobConfig.JobLocalDir); NUnit.Framework.Assert.IsNotNull(MRJobConfig.JobLocalDir + " is null", jobLocalDir ); }
private static string GetChildEnv(JobConf jobConf, bool isMap) { if (isMap) { return(jobConf.Get(JobConf.MapredMapTaskEnv, jobConf.Get(JobConf.MapredTaskEnv))); } return(jobConf.Get(JobConf.MapredReduceTaskEnv, jobConf.Get(JobConf.MapredTaskEnv ))); }
public virtual void Configure(JobConf job) { this.fieldSeparator = job.Get(FieldSelectionHelper.DataFieldSeperator, "\t"); this.mapOutputKeyValueSpec = job.Get(FieldSelectionHelper.MapOutputKeyValueSpec, "0-:"); this.ignoreInputKey = typeof(TextInputFormat).GetCanonicalName().Equals(job.GetInputFormat ().GetType().GetCanonicalName()); this.reduceOutputKeyValueSpec = job.Get(FieldSelectionHelper.ReduceOutputKeyValueSpec , "0-:"); ParseOutputKeyValueSpec(); Log.Info(SpecToString()); }
/// <exception cref="System.Exception"/> public virtual void TestMRConfig() { JobConf conf = CreateJobConf(); NUnit.Framework.Assert.IsNull(conf.Get("xyz")); Properties config = new Properties(); config.SetProperty("xyz", "XYZ"); StopCluster(); StartCluster(false, config); conf = CreateJobConf(); NUnit.Framework.Assert.AreEqual("XYZ", conf.Get("xyz")); }
/// <summary>test configuration for db.</summary> /// <remarks>test configuration for db. should works DBConfiguration.* parameters.</remarks> public virtual void TestSetInput() { JobConf configuration = new JobConf(); string[] fieldNames = new string[] { "field1", "field2" }; DBInputFormat.SetInput(configuration, typeof(DBInputFormat.NullDBWritable), "table" , "conditions", "orderBy", fieldNames); NUnit.Framework.Assert.AreEqual("org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable" , configuration.GetClass(DBConfiguration.InputClassProperty, null).FullName); NUnit.Framework.Assert.AreEqual("table", configuration.Get(DBConfiguration.InputTableNameProperty , null)); string[] fields = configuration.GetStrings(DBConfiguration.InputFieldNamesProperty ); NUnit.Framework.Assert.AreEqual("field1", fields[0]); NUnit.Framework.Assert.AreEqual("field2", fields[1]); NUnit.Framework.Assert.AreEqual("conditions", configuration.Get(DBConfiguration.InputConditionsProperty , null)); NUnit.Framework.Assert.AreEqual("orderBy", configuration.Get(DBConfiguration.InputOrderByProperty , null)); configuration = new JobConf(); DBInputFormat.SetInput(configuration, typeof(DBInputFormat.NullDBWritable), "query" , "countQuery"); NUnit.Framework.Assert.AreEqual("query", configuration.Get(DBConfiguration.InputQuery , null)); NUnit.Framework.Assert.AreEqual("countQuery", configuration.Get(DBConfiguration.InputCountQuery , null)); JobConf jConfiguration = new JobConf(); DBConfiguration.ConfigureDB(jConfiguration, "driverClass", "dbUrl", "user", "password" ); NUnit.Framework.Assert.AreEqual("driverClass", jConfiguration.Get(DBConfiguration .DriverClassProperty)); NUnit.Framework.Assert.AreEqual("dbUrl", jConfiguration.Get(DBConfiguration.UrlProperty )); NUnit.Framework.Assert.AreEqual("user", jConfiguration.Get(DBConfiguration.UsernameProperty )); NUnit.Framework.Assert.AreEqual("password", jConfiguration.Get(DBConfiguration.PasswordProperty )); jConfiguration = new JobConf(); DBConfiguration.ConfigureDB(jConfiguration, "driverClass", "dbUrl"); NUnit.Framework.Assert.AreEqual("driverClass", jConfiguration.Get(DBConfiguration .DriverClassProperty)); NUnit.Framework.Assert.AreEqual("dbUrl", jConfiguration.Get(DBConfiguration.UrlProperty )); NUnit.Framework.Assert.IsNull(jConfiguration.Get(DBConfiguration.UsernameProperty )); NUnit.Framework.Assert.IsNull(jConfiguration.Get(DBConfiguration.PasswordProperty )); }
public override void Configure(JobConf conf) { try { string taskId = conf.Get(JobContext.TaskAttemptId); id = System.Convert.ToInt32(taskId.Split("_")[4]); totalMaps = System.Convert.ToInt32(conf.Get(JobContext.NumMaps)); fs = FileSystem.Get(conf); signal = new Path(conf.Get(GetTaskSignalParameter(true))); } catch (IOException) { System.Console.Out.WriteLine("Got an exception while obtaining the filesystem"); } }
/// <summary> /// Get the /// <see cref="Org.Apache.Hadoop.IO.SequenceFile.CompressionType"/> /// for the output /// <see cref="Org.Apache.Hadoop.IO.SequenceFile"/> /// . /// </summary> /// <param name="conf"> /// the /// <see cref="JobConf"/> /// </param> /// <returns> /// the /// <see cref="Org.Apache.Hadoop.IO.SequenceFile.CompressionType"/> /// for the output /// <see cref="Org.Apache.Hadoop.IO.SequenceFile"/> /// , /// defaulting to /// <see cref="Org.Apache.Hadoop.IO.SequenceFile.CompressionType.Record"/> /// </returns> public static SequenceFile.CompressionType GetOutputCompressionType(JobConf conf) { string val = conf.Get(FileOutputFormat.CompressType, SequenceFile.CompressionType .Record.ToString()); return(SequenceFile.CompressionType.ValueOf(val)); }
/// <exception cref="System.IO.IOException"/> internal static long ReadBench(JobConf conf) { // InputFormat instantiation InputFormat inf = conf.GetInputFormat(); string fn = conf.Get("test.filebench.name", string.Empty); Path pin = new Path(FileInputFormat.GetInputPaths(conf)[0], fn); FileStatus @in = pin.GetFileSystem(conf).GetFileStatus(pin); RecordReader rr = inf.GetRecordReader(new FileSplit(pin, 0, @in.GetLen(), (string [])null), conf, Reporter.Null); try { object key = rr.CreateKey(); object val = rr.CreateValue(); DateTime start = new DateTime(); while (rr.Next(key, val)) { } DateTime end = new DateTime(); return(end.GetTime() - start.GetTime()); } finally { rr.Close(); } }
/// <exception cref="System.IO.IOException"/> public override RecordWriter <K, V> GetRecordWriter(FileSystem ignored, JobConf job , string name, Progressable progress) { bool isCompressed = GetCompressOutput(job); string keyValueSeparator = job.Get("mapreduce.output.textoutputformat.separator", "\t"); if (!isCompressed) { Path file = FileOutputFormat.GetTaskOutputPath(job, name); FileSystem fs = file.GetFileSystem(job); FSDataOutputStream fileOut = fs.Create(file, progress); return(new TextOutputFormat.LineRecordWriter <K, V>(fileOut, keyValueSeparator)); } else { Type codecClass = GetOutputCompressorClass(job, typeof(GzipCodec)); // create the named codec CompressionCodec codec = ReflectionUtils.NewInstance(codecClass, job); // build the filename including the extension Path file = FileOutputFormat.GetTaskOutputPath(job, name + codec.GetDefaultExtension ()); FileSystem fs = file.GetFileSystem(job); FSDataOutputStream fileOut = fs.Create(file, progress); return(new TextOutputFormat.LineRecordWriter <K, V>(new DataOutputStream(codec.CreateOutputStream (fileOut)), keyValueSeparator)); } }
/// <summary> /// Set the configuration, if it doesn't already have a value for the given /// key. /// </summary> /// <param name="conf">the configuration to modify</param> /// <param name="key">the key to set</param> /// <param name="value">the new "default" value to set</param> private static void SetIfUnset(JobConf conf, string key, string value) { if (conf.Get(key) == null) { conf.Set(key, value); } }
private Configuration AddSecurityConfiguration(Configuration conf) { conf = new JobConf(conf); conf.Set(CommonConfigurationKeys.HadoopSecurityServiceUserNameKey, conf.Get(JHAdminConfig .MrHistoryPrincipal, string.Empty)); return(conf); }
/// <exception cref="System.IO.IOException"/> public virtual void TestWithDFS() { MiniDFSCluster dfs = null; MiniMRCluster mr = null; FileSystem fileSys = null; try { int taskTrackers = 4; JobConf conf = new JobConf(); conf.Set(JTConfig.JtSystemDir, "/tmp/custom/mapred/system"); dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build(); fileSys = dfs.GetFileSystem(); mr = new MiniMRCluster(taskTrackers, fileSys.GetUri().ToString(), 1, null, null, conf); RunWordCount(mr, mr.CreateJobConf(), conf.Get("mapred.system.dir")); } finally { if (dfs != null) { dfs.Shutdown(); } if (mr != null) { mr.Shutdown(); } } }
/// <exception cref="System.Exception"/> private string ReadStdOut(JobConf conf) { TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName(conf.Get(MRJobConfig .TaskAttemptId))); FilePath stdOut = TaskLog.GetTaskLogFile(taskId, false, TaskLog.LogName.Stdout); return(ReadFile(stdOut)); }
public virtual void TestProfileParamsSetter() { JobConf configuration = new JobConf(); configuration.SetProfileParams("test"); NUnit.Framework.Assert.AreEqual("test", configuration.Get(MRJobConfig.TaskProfileParams )); }
private static IntWritable DeduceInputFile(JobConf job) { Path[] inputPaths = FileInputFormat.GetInputPaths(job); Path inputFile = new Path(job.Get(JobContext.MapInputFile)); // value == one for sort-input; value == two for sort-output return((inputFile.GetParent().Equals(inputPaths[0])) ? sortInput : sortOutput); }
/// <summary> /// Add a /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// to the list of inputs for the map-reduce job. /// </summary> /// <param name="conf">The configuration of the job</param> /// <param name="path"> /// /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// to be added to the list of inputs for /// the map-reduce job. /// </param> public static void AddInputPath(JobConf conf, Path path) { path = new Path(conf.GetWorkingDirectory(), path); string dirStr = StringUtils.EscapeString(path.ToString()); string dirs = conf.Get(FileInputFormat.InputDir); conf.Set(FileInputFormat.InputDir, dirs == null ? dirStr : dirs + StringUtils.CommaStr + dirStr); }
/// <summary> /// Add a /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// with a custom /// <see cref="Org.Apache.Hadoop.Mapred.InputFormat{K, V}"/> /// to the list of /// inputs for the map-reduce job. /// </summary> /// <param name="conf">The configuration of the job</param> /// <param name="path"> /// /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// to be added to the list of inputs for the job /// </param> /// <param name="inputFormatClass"> /// /// <see cref="Org.Apache.Hadoop.Mapred.InputFormat{K, V}"/> /// class to use for this path /// </param> public static void AddInputPath(JobConf conf, Path path, Type inputFormatClass) { string inputFormatMapping = path.ToString() + ";" + inputFormatClass.FullName; string inputFormats = conf.Get("mapreduce.input.multipleinputs.dir.formats"); conf.Set("mapreduce.input.multipleinputs.dir.formats", inputFormats == null ? inputFormatMapping : inputFormats + "," + inputFormatMapping); conf.SetInputFormat(typeof(DelegatingInputFormat)); }
/// <exception cref="System.IO.IOException"/> internal static void CheckRecords(Configuration defaults, int noMaps, int noReduces , Path sortInput, Path sortOutput) { JobConf jobConf = new JobConf(defaults, typeof(SortValidator.RecordChecker)); jobConf.SetJobName("sortvalidate-record-checker"); jobConf.SetInputFormat(typeof(SequenceFileInputFormat)); jobConf.SetOutputFormat(typeof(SequenceFileOutputFormat)); jobConf.SetOutputKeyClass(typeof(BytesWritable)); jobConf.SetOutputValueClass(typeof(IntWritable)); jobConf.SetMapperClass(typeof(SortValidator.RecordChecker.Map)); jobConf.SetReducerClass(typeof(SortValidator.RecordChecker.Reduce)); JobClient client = new JobClient(jobConf); ClusterStatus cluster = client.GetClusterStatus(); if (noMaps == -1) { noMaps = cluster.GetTaskTrackers() * jobConf.GetInt(MapsPerHost, 10); } if (noReduces == -1) { noReduces = (int)(cluster.GetMaxReduceTasks() * 0.9); string sortReduces = jobConf.Get(ReducesPerHost); if (sortReduces != null) { noReduces = cluster.GetTaskTrackers() * System.Convert.ToInt32(sortReduces); } } jobConf.SetNumMapTasks(noMaps); jobConf.SetNumReduceTasks(noReduces); FileInputFormat.SetInputPaths(jobConf, sortInput); FileInputFormat.AddInputPath(jobConf, sortOutput); Path outputPath = new Path("/tmp/sortvalidate/recordchecker"); FileSystem fs = FileSystem.Get(defaults); if (fs.Exists(outputPath)) { fs.Delete(outputPath, true); } FileOutputFormat.SetOutputPath(jobConf, outputPath); // Uncomment to run locally in a single process //job_conf.set(JTConfig.JT, "local"); Path[] inputPaths = FileInputFormat.GetInputPaths(jobConf); System.Console.Out.WriteLine("\nSortValidator.RecordChecker: Running on " + cluster .GetTaskTrackers() + " nodes to validate sort from " + inputPaths[0] + ", " + inputPaths [1] + " into " + FileOutputFormat.GetOutputPath(jobConf) + " with " + noReduces + " reduces."); DateTime startTime = new DateTime(); System.Console.Out.WriteLine("Job started: " + startTime); JobClient.RunJob(jobConf); DateTime end_time = new DateTime(); System.Console.Out.WriteLine("Job ended: " + end_time); System.Console.Out.WriteLine("The job took " + (end_time.GetTime() - startTime.GetTime ()) / 1000 + " seconds."); }
public virtual void TestSocketFactory() { // Create a standard mini-cluster Configuration sconf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(sconf).NumDataNodes(1).Build( ); int nameNodePort = cluster.GetNameNodePort(); // Get a reference to its DFS directly FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem); DistributedFileSystem directDfs = (DistributedFileSystem)fs; Configuration cconf = GetCustomSocketConfigs(nameNodePort); fs = FileSystem.Get(cconf); NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem); DistributedFileSystem dfs = (DistributedFileSystem)fs; JobClient client = null; MiniMRYarnCluster miniMRYarnCluster = null; try { // This will test RPC to the NameNode only. // could we test Client-DataNode connections? Path filePath = new Path("/dir"); NUnit.Framework.Assert.IsFalse(directDfs.Exists(filePath)); NUnit.Framework.Assert.IsFalse(dfs.Exists(filePath)); directDfs.Mkdirs(filePath); NUnit.Framework.Assert.IsTrue(directDfs.Exists(filePath)); NUnit.Framework.Assert.IsTrue(dfs.Exists(filePath)); // This will test RPC to a Resource Manager fs = FileSystem.Get(sconf); JobConf jobConf = new JobConf(); FileSystem.SetDefaultUri(jobConf, fs.GetUri().ToString()); miniMRYarnCluster = InitAndStartMiniMRYarnCluster(jobConf); JobConf jconf = new JobConf(miniMRYarnCluster.GetConfig()); jconf.Set("hadoop.rpc.socket.factory.class.default", "org.apache.hadoop.ipc.DummySocketFactory" ); jconf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); string rmAddress = jconf.Get("yarn.resourcemanager.address"); string[] split = rmAddress.Split(":"); jconf.Set("yarn.resourcemanager.address", split[0] + ':' + (System.Convert.ToInt32 (split[1]) + 10)); client = new JobClient(jconf); JobStatus[] jobs = client.JobsToComplete(); NUnit.Framework.Assert.IsTrue(jobs.Length == 0); } finally { CloseClient(client); CloseDfs(dfs); CloseDfs(directDfs); StopMiniMRYarnCluster(miniMRYarnCluster); ShutdownDFSCluster(cluster); } }
/// <summary> /// Add a /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// with a custom /// <see cref="Org.Apache.Hadoop.Mapred.InputFormat{K, V}"/> /// and /// <see cref="Org.Apache.Hadoop.Mapred.Mapper{K1, V1, K2, V2}"/> /// to the list of inputs for the map-reduce job. /// </summary> /// <param name="conf">The configuration of the job</param> /// <param name="path"> /// /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// to be added to the list of inputs for the job /// </param> /// <param name="inputFormatClass"> /// /// <see cref="Org.Apache.Hadoop.Mapred.InputFormat{K, V}"/> /// class to use for this path /// </param> /// <param name="mapperClass"> /// /// <see cref="Org.Apache.Hadoop.Mapred.Mapper{K1, V1, K2, V2}"/> /// class to use for this path /// </param> public static void AddInputPath(JobConf conf, Path path, Type inputFormatClass, Type mapperClass) { AddInputPath(conf, path, inputFormatClass); string mapperMapping = path.ToString() + ";" + mapperClass.FullName; string mappers = conf.Get("mapreduce.input.multipleinputs.dir.mappers"); conf.Set("mapreduce.input.multipleinputs.dir.mappers", mappers == null ? mapperMapping : mappers + "," + mapperMapping); conf.SetMapperClass(typeof(DelegatingMapper)); }
/// <summary> /// Get the list of input /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// s for the map-reduce job. /// </summary> /// <param name="conf">The configuration of the job</param> /// <returns> /// the list of input /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// s for the map-reduce job. /// </returns> public static Path[] GetInputPaths(JobConf conf) { string dirs = conf.Get(FileInputFormat.InputDir, string.Empty); string[] list = StringUtils.Split(dirs); Path[] result = new Path[list.Length]; for (int i = 0; i < list.Length; i++) { result[i] = new Path(StringUtils.UnEscapeString(list[i])); } return(result); }
public override void Configure(JobConf conf) { try { fs = FileSystem.Get(conf); signal = new Path(conf.Get(GetTaskSignalParameter(false))); } catch (IOException) { System.Console.Out.WriteLine("Got an exception while obtaining the filesystem"); } }
/// <summary>Adds a named output for the job.</summary> /// <param name="conf">job conf to add the named output</param> /// <param name="namedOutput"> /// named output name, it has to be a word, letters /// and numbers only, cannot be the word 'part' as /// that is reserved for the /// default output. /// </param> /// <param name="multi">indicates if the named output is multi</param> /// <param name="outputFormatClass">OutputFormat class.</param> /// <param name="keyClass">key class</param> /// <param name="valueClass">value class</param> private static void AddNamedOutput(JobConf conf, string namedOutput, bool multi, Type outputFormatClass, Type keyClass, Type valueClass) { CheckNamedOutputName(namedOutput); CheckNamedOutput(conf, namedOutput, true); conf.Set(NamedOutputs, conf.Get(NamedOutputs, string.Empty) + " " + namedOutput); conf.SetClass(MoPrefix + namedOutput + Format, outputFormatClass, typeof(OutputFormat )); conf.SetClass(MoPrefix + namedOutput + Key, keyClass, typeof(object)); conf.SetClass(MoPrefix + namedOutput + Value, valueClass, typeof(object)); conf.SetBoolean(MoPrefix + namedOutput + Multi, multi); }
/// <summary>Returns list of channel names.</summary> /// <param name="conf">job conf</param> /// <returns>List of channel Names</returns> public static IList <string> GetNamedOutputsList(JobConf conf) { IList <string> names = new AList <string>(); StringTokenizer st = new StringTokenizer(conf.Get(NamedOutputs, string.Empty), " " ); while (st.HasMoreTokens()) { names.AddItem(st.NextToken()); } return(names); }
public virtual void Configure(JobConf conf) { this.conf = conf; NUnit.Framework.Assert.AreEqual(prop, conf.Get("a")); try { WriteFlag(conf, "configure." + name); } catch (IOException ex) { throw new RuntimeException(ex); } }
/// <exception cref="System.IO.IOException"/> public override RecordReader <LongWritable, Text> GetRecordReader(InputSplit genericSplit , JobConf job, Reporter reporter) { reporter.SetStatus(genericSplit.ToString()); string delimiter = job.Get("textinputformat.record.delimiter"); byte[] recordDelimiterBytes = null; if (null != delimiter) { recordDelimiterBytes = Sharpen.Runtime.GetBytesForString(delimiter, Charsets.Utf8 ); } return(new LineRecordReader(job, (FileSplit)genericSplit, recordDelimiterBytes)); }
/// <exception cref="System.Exception"/> public virtual void TestNoDefaults() { JobConf configuration = new JobConf(); NUnit.Framework.Assert.IsTrue(configuration.Get("hadoop.tmp.dir", null) != null); configuration = new JobConf(false); NUnit.Framework.Assert.IsTrue(configuration.Get("hadoop.tmp.dir", null) == null); Path inDir = new Path("testing/jobconf/input"); Path outDir = new Path("testing/jobconf/output"); OutputStream os = GetFileSystem().Create(new Path(inDir, "text.txt")); TextWriter wr = new OutputStreamWriter(os); wr.Write("hello\n"); wr.Write("hello\n"); wr.Close(); JobConf conf = new JobConf(false); conf.Set("fs.defaultFS", CreateJobConf().Get("fs.defaultFS")); conf.SetJobName("mr"); conf.SetInputFormat(typeof(TextInputFormat)); conf.SetMapOutputKeyClass(typeof(LongWritable)); conf.SetMapOutputValueClass(typeof(Text)); conf.SetOutputFormat(typeof(TextOutputFormat)); conf.SetOutputKeyClass(typeof(LongWritable)); conf.SetOutputValueClass(typeof(Text)); conf.SetMapperClass(typeof(IdentityMapper)); conf.SetReducerClass(typeof(IdentityReducer)); FileInputFormat.SetInputPaths(conf, inDir); FileOutputFormat.SetOutputPath(conf, outDir); JobClient.RunJob(conf); Path[] outputFiles = FileUtil.Stat2Paths(GetFileSystem().ListStatus(outDir, new Utils.OutputFileUtils.OutputFilesFilter ())); if (outputFiles.Length > 0) { InputStream @is = GetFileSystem().Open(outputFiles[0]); BufferedReader reader = new BufferedReader(new InputStreamReader(@is)); string line = reader.ReadLine(); int counter = 0; while (line != null) { counter++; NUnit.Framework.Assert.IsTrue(line.Contains("hello")); line = reader.ReadLine(); } reader.Close(); NUnit.Framework.Assert.AreEqual(2, counter); } }