/// <exception cref="System.IO.IOException"/> public virtual void TestClose() { Configuration conf = new Configuration(); LocalFileSystem fs = FileSystem.GetLocal(conf); // create a sequence file 1 Path path1 = new Path(Runtime.GetProperty("test.build.data", ".") + "/test1.seq"); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, path1, typeof(Text ), typeof(NullWritable), SequenceFile.CompressionType.Block); writer.Append(new Text("file1-1"), NullWritable.Get()); writer.Append(new Text("file1-2"), NullWritable.Get()); writer.Close(); Path path2 = new Path(Runtime.GetProperty("test.build.data", ".") + "/test2.seq"); writer = SequenceFile.CreateWriter(fs, conf, path2, typeof(Text), typeof(NullWritable ), SequenceFile.CompressionType.Block); writer.Append(new Text("file2-1"), NullWritable.Get()); writer.Append(new Text("file2-2"), NullWritable.Get()); writer.Close(); // Create a reader which uses 4 BuiltInZLibInflater instances SequenceFile.Reader reader = new SequenceFile.Reader(fs, path1, conf); // Returns the 4 BuiltInZLibInflater instances to the CodecPool reader.Close(); // The second close _could_ erroneously returns the same // 4 BuiltInZLibInflater instances to the CodecPool again reader.Close(); // The first reader gets 4 BuiltInZLibInflater instances from the CodecPool SequenceFile.Reader reader1 = new SequenceFile.Reader(fs, path1, conf); // read first value from reader1 Text text = new Text(); reader1.Next(text); Assert.Equal("file1-1", text.ToString()); // The second reader _could_ get the same 4 BuiltInZLibInflater // instances from the CodePool as reader1 SequenceFile.Reader reader2 = new SequenceFile.Reader(fs, path2, conf); // read first value from reader2 reader2.Next(text); Assert.Equal("file2-1", text.ToString()); // read second value from reader1 reader1.Next(text); Assert.Equal("file1-2", text.ToString()); // read second value from reader2 (this throws an exception) reader2.Next(text); Assert.Equal("file2-2", text.ToString()); NUnit.Framework.Assert.IsFalse(reader1.Next(text)); NUnit.Framework.Assert.IsFalse(reader2.Next(text)); }
/// <exception cref="System.IO.IOException"/> private static void CreateBigMapInputFile(Configuration conf, FileSystem fs, Path dir, long fileSizeInMB) { // Check if the input path exists and is non-empty if (fs.Exists(dir)) { FileStatus[] list = fs.ListStatus(dir); if (list.Length > 0) { throw new IOException("Input path: " + dir + " already exists... "); } } Path file = new Path(dir, "part-0"); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(BytesWritable ), typeof(BytesWritable), SequenceFile.CompressionType.None); long numBytesToWrite = fileSizeInMB * 1024 * 1024; int minKeySize = conf.GetInt(MinKey, 10); int keySizeRange = conf.GetInt(MaxKey, 1000) - minKeySize; int minValueSize = conf.GetInt(MinValue, 0); int valueSizeRange = conf.GetInt(MaxValue, 20000) - minValueSize; BytesWritable randomKey = new BytesWritable(); BytesWritable randomValue = new BytesWritable(); Log.Info("Writing " + numBytesToWrite + " bytes to " + file + " with " + "minKeySize: " + minKeySize + " keySizeRange: " + keySizeRange + " minValueSize: " + minValueSize + " valueSizeRange: " + valueSizeRange); long start = Runtime.CurrentTimeMillis(); while (numBytesToWrite > 0) { int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0); randomKey.SetSize(keyLength); RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength()); int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange ) : 0); randomValue.SetSize(valueLength); RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength()); writer.Append(randomKey, randomValue); numBytesToWrite -= keyLength + valueLength; } writer.Close(); long end = Runtime.CurrentTimeMillis(); Log.Info("Created " + file + " of size: " + fileSizeInMB + "MB in " + (end - start ) / 1000 + "secs"); }
/// <exception cref="System.IO.IOException"/> public virtual void RunTest(SequenceFile.CompressionType compressionType) { JobConf job = new JobConf(); FileSystem fs = FileSystem.GetLocal(job); Path dir = new Path(Runtime.GetProperty("test.build.data", ".") + "/mapred"); Path file = new Path(dir, "test.seq"); Path tempDir = new Path(dir, "tmp"); fs.Delete(dir, true); FileInputFormat.SetInputPaths(job, dir); fs.Mkdirs(tempDir); LongWritable tkey = new LongWritable(); Text tval = new Text(); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, job, file, typeof(LongWritable ), typeof(Text), compressionType, new DefaultCodec()); try { for (int i = 0; i < Records; ++i) { tkey.Set(1234); tval.Set("valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue"); writer.Append(tkey, tval); } } finally { writer.Close(); } long fileLength = fs.GetFileStatus(file).GetLen(); Log.Info("With compression = " + compressionType + ": " + "compressed length = " + fileLength); SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, job.GetOutputKeyComparator (), job.GetMapOutputKeyClass(), job.GetMapOutputValueClass(), job); Path[] paths = new Path[] { file }; SequenceFile.Sorter.RawKeyValueIterator rIter = sorter.Merge(paths, tempDir, false ); int count = 0; while (rIter.Next()) { count++; } NUnit.Framework.Assert.AreEqual(Records, count); NUnit.Framework.Assert.AreEqual(1.0f, rIter.GetProgress().Get()); }
/// <exception cref="System.IO.IOException"/> private void WriteMetadataTest(FileSystem fs, int count, int seed, Path file, SequenceFile.CompressionType compressionType, CompressionCodec codec, SequenceFile.Metadata metadata) { fs.Delete(file, true); Log.Info("creating " + count + " records with metadata and with " + compressionType + " compression"); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(RandomDatum ), typeof(RandomDatum), compressionType, codec, null, metadata); RandomDatum.Generator generator = new RandomDatum.Generator(seed); for (int i = 0; i < count; i++) { generator.Next(); RandomDatum key = generator.GetKey(); RandomDatum value = generator.GetValue(); writer.Append(key, value); } writer.Close(); }
public virtual void Configure() { Path testdir = new Path(TestDir.GetAbsolutePath()); Path inDir = new Path(testdir, "in"); Path outDir = new Path(testdir, "out"); FileSystem fs = FileSystem.Get(conf); fs.Delete(testdir, true); conf.SetInputFormat(typeof(SequenceFileInputFormat)); FileInputFormat.SetInputPaths(conf, inDir); FileOutputFormat.SetOutputPath(conf, outDir); conf.SetOutputKeyClass(typeof(IntWritable)); conf.SetOutputValueClass(typeof(Text)); conf.SetMapOutputValueClass(typeof(IntWritable)); // set up two map jobs, so we can test merge phase in Reduce also conf.SetNumMapTasks(2); conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName); conf.SetOutputFormat(typeof(SequenceFileOutputFormat)); if (!fs.Mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.ToString()); } if (!fs.Mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir.ToString()); } // set up input data in 2 files Path inFile = new Path(inDir, "part0"); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, inFile, typeof(IntWritable ), typeof(IntWritable)); writer.Append(new IntWritable(11), new IntWritable(999)); writer.Append(new IntWritable(23), new IntWritable(456)); writer.Append(new IntWritable(10), new IntWritable(780)); writer.Close(); inFile = new Path(inDir, "part1"); writer = SequenceFile.CreateWriter(fs, conf, inFile, typeof(IntWritable), typeof( IntWritable)); writer.Append(new IntWritable(45), new IntWritable(100)); writer.Append(new IntWritable(18), new IntWritable(200)); writer.Append(new IntWritable(27), new IntWritable(300)); writer.Close(); jc = new JobClient(conf); }
// clean up after all to restore the system state /// <exception cref="System.IO.IOException"/> private void CreateInputFile(string rootName) { Cleanup(); // clean up if previous run failed Path inputFile = new Path(MapInputDir, "in_file"); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, fsConfig, inputFile, typeof( Text), typeof(LongWritable), SequenceFile.CompressionType.None); try { nrFiles = 0; ListSubtree(new Path(rootName), writer); } finally { writer.Close(); } Log.Info("Created map input files."); }
/// <exception cref="System.Exception"/> public virtual void TestJavaSerialization() { Path file = new Path(Runtime.GetProperty("test.build.data", ".") + "/testseqser.seq" ); fs.Delete(file, true); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(long ), typeof(string)); writer.Append(1L, "one"); writer.Append(2L, "two"); writer.Close(); SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf); Assert.Equal(1L, reader.Next((object)null)); Assert.Equal("one", reader.GetCurrentValue((object)null)); Assert.Equal(2L, reader.Next((object)null)); Assert.Equal("two", reader.GetCurrentValue((object)null)); NUnit.Framework.Assert.IsNull(reader.Next((object)null)); reader.Close(); }
/// <exception cref="System.Exception"/> private static void CreateSequenceFile(int numRecords) { // create a file with length entries SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, inFile, typeof(Text ), typeof(BytesWritable)); try { for (int i = 1; i <= numRecords; i++) { Text key = new Text(Sharpen.Extensions.ToString(i)); byte[] data = new byte[random.Next(10)]; random.NextBytes(data); BytesWritable value = new BytesWritable(data); writer.Append(key, value); } } finally { writer.Close(); } }
/// <exception cref="System.IO.IOException"/> public override RecordWriter <BytesWritable, BytesWritable> GetRecordWriter(FileSystem ignored, JobConf job, string name, Progressable progress) { // get the path of the temporary output file Path file = FileOutputFormat.GetTaskOutputPath(job, name); FileSystem fs = file.GetFileSystem(job); CompressionCodec codec = null; SequenceFile.CompressionType compressionType = SequenceFile.CompressionType.None; if (GetCompressOutput(job)) { // find the kind of compression to do compressionType = GetOutputCompressionType(job); // find the right codec Type codecClass = GetOutputCompressorClass(job, typeof(DefaultCodec)); codec = ReflectionUtils.NewInstance(codecClass, job); } SequenceFile.Writer @out = SequenceFile.CreateWriter(fs, job, file, GetSequenceFileOutputKeyClass (job), GetSequenceFileOutputValueClass(job), compressionType, codec, progress); return(new _RecordWriter_138(@out)); }
/// <exception cref="System.IO.IOException"/> protected internal virtual SequenceFile.Writer GetSequenceWriter(TaskAttemptContext context, Type keyClass, Type valueClass) { Configuration conf = context.GetConfiguration(); CompressionCodec codec = null; SequenceFile.CompressionType compressionType = SequenceFile.CompressionType.None; if (GetCompressOutput(context)) { // find the kind of compression to do compressionType = GetOutputCompressionType(context); // find the right codec Type codecClass = GetOutputCompressorClass(context, typeof(DefaultCodec)); codec = (CompressionCodec)ReflectionUtils.NewInstance(codecClass, conf); } // get the path of the temporary output file Path file = GetDefaultWorkFile(context, string.Empty); FileSystem fs = file.GetFileSystem(conf); return(SequenceFile.CreateWriter(fs, conf, file, keyClass, valueClass, compressionType , codec, context)); }
/// <exception cref="System.IO.IOException"/> public SeqFileAppendable(FileSystem fs, Path path, int osBufferSize, string compress , int minBlkSize) { Configuration conf = new Configuration(); conf.SetBoolean("hadoop.native.lib", true); CompressionCodec codec = null; if ("lzo".Equals(compress)) { codec = Compression.Algorithm.Lzo.GetCodec(); } else { if ("gz".Equals(compress)) { codec = Compression.Algorithm.Gz.GetCodec(); } else { if (!"none".Equals(compress)) { throw new IOException("Codec not supported."); } } } this.fsdos = fs.Create(path, true, osBufferSize); if (!"none".Equals(compress)) { writer = SequenceFile.CreateWriter(conf, fsdos, typeof(BytesWritable), typeof(BytesWritable ), SequenceFile.CompressionType.Block, codec); } else { writer = SequenceFile.CreateWriter(conf, fsdos, typeof(BytesWritable), typeof(BytesWritable ), SequenceFile.CompressionType.None, null); } }
/// <summary>Write a partition file for the given job, using the Sampler provided.</summary> /// <remarks> /// Write a partition file for the given job, using the Sampler provided. /// Queries the sampler for a sample keyset, sorts by the output key /// comparator, selects the keys for each rank, and writes to the destination /// returned from /// <see cref="TotalOrderPartitioner{K, V}.GetPartitionFile(Org.Apache.Hadoop.Conf.Configuration) /// "/> /// . /// </remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.TypeLoadException"/> /// <exception cref="System.Exception"/> public static void WritePartitionFile <K, V>(Job job, InputSampler.Sampler <K, V> sampler ) { // getInputFormat, getOutputKeyComparator Configuration conf = job.GetConfiguration(); InputFormat inf = ReflectionUtils.NewInstance(job.GetInputFormatClass(), conf); int numPartitions = job.GetNumReduceTasks(); K[] samples = (K[])sampler.GetSample(inf, job); Log.Info("Using " + samples.Length + " samples"); RawComparator <K> comparator = (RawComparator <K>)job.GetSortComparator(); Arrays.Sort(samples, comparator); Path dst = new Path(TotalOrderPartitioner.GetPartitionFile(conf)); FileSystem fs = dst.GetFileSystem(conf); if (fs.Exists(dst)) { fs.Delete(dst, false); } SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, dst, job.GetMapOutputKeyClass (), typeof(NullWritable)); NullWritable nullValue = NullWritable.Get(); float stepSize = samples.Length / (float)numPartitions; int last = -1; for (int i = 1; i < numPartitions; ++i) { int k = Math.Round(stepSize * i); while (last >= k && comparator.Compare(samples[last], samples[k]) == 0) { ++k; } writer.Append(samples[k], nullValue); last = k; } writer.Close(); }
/// <exception cref="System.IO.IOException"/> public virtual void TestRecursiveSeqFileCreate() { FileSystem fs = FileSystem.GetLocal(conf); Path name = new Path(new Path(Runtime.GetProperty("test.build.data", "."), "recursiveCreateDir" ), "file"); bool createParent = false; try { SequenceFile.CreateWriter(fs, conf, name, typeof(RandomDatum), typeof(RandomDatum ), 512, (short)1, 4096, createParent, SequenceFile.CompressionType.None, null, new SequenceFile.Metadata()); Fail("Expected an IOException due to missing parent"); } catch (IOException) { } // Expected createParent = true; SequenceFile.CreateWriter(fs, conf, name, typeof(RandomDatum), typeof(RandomDatum ), 512, (short)1, 4096, createParent, SequenceFile.CompressionType.None, null, new SequenceFile.Metadata()); }
/// <exception cref="System.IO.IOException"/> internal virtual void CreateTempFile(Path p, Configuration conf) { SequenceFile.Writer writer = null; try { writer = SequenceFile.CreateWriter(fs, conf, p, typeof(Text), typeof(Text), SequenceFile.CompressionType .None); writer.Append(new Text("text"), new Text("moretext")); } catch (Exception e) { throw new IOException(e.GetLocalizedMessage()); } finally { if (writer != null) { writer.Close(); } writer = null; } Log.Info("created: " + p); }
/// <exception cref="System.IO.IOException"/> private void MergeTest(FileSystem fs, int count, int seed, Path file, SequenceFile.CompressionType compressionType, bool fast, int factor, int megabytes) { Log.Debug("creating " + factor + " files with " + count / factor + " records"); SequenceFile.Writer[] writers = new SequenceFile.Writer[factor]; Path[] names = new Path[factor]; Path[] sortedNames = new Path[factor]; for (int i = 0; i < factor; i++) { names[i] = file.Suffix("." + i); sortedNames[i] = names[i].Suffix(".sorted"); fs.Delete(names[i], true); fs.Delete(sortedNames[i], true); writers[i] = SequenceFile.CreateWriter(fs, conf, names[i], typeof(RandomDatum), typeof( RandomDatum), compressionType); } RandomDatum.Generator generator = new RandomDatum.Generator(seed); for (int i_1 = 0; i_1 < count; i_1++) { generator.Next(); RandomDatum key = generator.GetKey(); RandomDatum value = generator.GetValue(); writers[i_1 % factor].Append(key, value); } for (int i_2 = 0; i_2 < factor; i_2++) { writers[i_2].Close(); } for (int i_3 = 0; i_3 < factor; i_3++) { Log.Debug("sorting file " + i_3 + " with " + count / factor + " records"); NewSorter(fs, fast, megabytes, factor).Sort(names[i_3], sortedNames[i_3]); } Log.Info("merging " + factor + " files with " + count / factor + " debug"); fs.Delete(new Path(file + ".sorted"), true); NewSorter(fs, fast, megabytes, factor).Merge(sortedNames, file.Suffix(".sorted")); }
/// <exception cref="System.Exception"/> public static void CreateControlFile(FileSystem fs, long megaBytes, int numFiles, long seed) { Log.Info("creating control file: " + megaBytes + " bytes, " + numFiles + " files" ); Path controlFile = new Path(ControlDir, "files"); fs.Delete(controlFile, true); Random random = new Random(seed); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, controlFile, typeof( Text), typeof(LongWritable), SequenceFile.CompressionType.None); long totalSize = 0; long maxSize = ((megaBytes / numFiles) * 2) + 1; try { while (totalSize < megaBytes) { Text name = new Text(System.Convert.ToString(random.NextLong())); long size = random.NextLong(); if (size < 0) { size = -size; } size = size % maxSize; //LOG.info(" adding: name="+name+" size="+size); writer.Append(name, new LongWritable(size)); totalSize += size; } } finally { writer.Close(); } Log.Info("created control file for: " + totalSize + " bytes"); }
/// <exception cref="System.Exception"/> public virtual void TestAppendRecordCompression() { GenericTestUtils.AssumeInNativeProfile(); Path file = new Path(RootPath, "testseqappendblockcompr.seq"); fs.Delete(file, true); SequenceFile.Writer.Option compressOption = SequenceFile.Writer.Compression(SequenceFile.CompressionType .Record, new GzipCodec()); SequenceFile.Writer writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer. File(file), SequenceFile.Writer.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass (typeof(string)), compressOption); writer.Append(1L, "one"); writer.Append(2L, "two"); writer.Close(); Verify2Values(file); writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer .KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer .AppendIfExists(true), compressOption); writer.Append(3L, "three"); writer.Append(4L, "four"); writer.Close(); VerifyAll4Values(file); fs.DeleteOnExit(file); }
/// <exception cref="System.Exception"/> public virtual int Run(string[] argv) { JobConf job = new JobConf(GetConf()); job.SetJarByClass(typeof(GenericMRLoadGenerator)); job.SetMapperClass(typeof(GenericMRLoadGenerator.SampleMapper)); job.SetReducerClass(typeof(GenericMRLoadGenerator.SampleReducer)); if (!ParseArgs(argv, job)) { return(-1); } if (null == FileOutputFormat.GetOutputPath(job)) { // No output dir? No writes job.SetOutputFormat(typeof(NullOutputFormat)); } if (0 == FileInputFormat.GetInputPaths(job).Length) { // No input dir? Generate random data System.Console.Error.WriteLine("No input path; ignoring InputFormat"); ConfRandom(job); } else { if (null != job.GetClass(GenericMRLoadGenerator.IndirectInputFormat, null)) { // specified IndirectInputFormat? Build src list JobClient jClient = new JobClient(job); Path tmpDir = new Path(jClient.GetFs().GetHomeDirectory(), ".staging"); Random r = new Random(); Path indirInputFile = new Path(tmpDir, Sharpen.Extensions.ToString(r.Next(int.MaxValue ), 36) + "_files"); job.Set(GenericMRLoadGenerator.IndirectInputFile, indirInputFile.ToString()); SequenceFile.Writer writer = SequenceFile.CreateWriter(tmpDir.GetFileSystem(job), job, indirInputFile, typeof(LongWritable), typeof(Text), SequenceFile.CompressionType .None); try { foreach (Path p in FileInputFormat.GetInputPaths(job)) { FileSystem fs = p.GetFileSystem(job); Stack <Path> pathstack = new Stack <Path>(); pathstack.Push(p); while (!pathstack.Empty()) { foreach (FileStatus stat in fs.ListStatus(pathstack.Pop())) { if (stat.IsDirectory()) { if (!stat.GetPath().GetName().StartsWith("_")) { pathstack.Push(stat.GetPath()); } } else { writer.Sync(); writer.Append(new LongWritable(stat.GetLen()), new Text(stat.GetPath().ToUri().ToString ())); } } } } } finally { writer.Close(); } } } DateTime startTime = new DateTime(); System.Console.Out.WriteLine("Job started: " + startTime); JobClient.RunJob(job); DateTime endTime = new DateTime(); System.Console.Out.WriteLine("Job ended: " + endTime); System.Console.Out.WriteLine("The job took " + (endTime.GetTime() - startTime.GetTime ()) / 1000 + " seconds."); return(0); }
/// <exception cref="System.Exception"/> public virtual void TestFormat() { JobConf job = new JobConf(conf); FileSystem fs = FileSystem.GetLocal(conf); Path dir = new Path(Runtime.GetProperty("test.build.data", ".") + "/mapred"); Path file = new Path(dir, "test.seq"); Reporter reporter = Reporter.Null; int seed = new Random().Next(); //LOG.info("seed = "+seed); Random random = new Random(seed); fs.Delete(dir, true); FileInputFormat.SetInputPaths(job, dir); // for a variety of lengths for (int length = 0; length < MaxLength; length += random.Next(MaxLength / 10) + 1) { //LOG.info("creating; entries = " + length); // create a file with length entries SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(IntWritable ), typeof(LongWritable)); try { for (int i = 0; i < length; i++) { IntWritable key = new IntWritable(i); LongWritable value = new LongWritable(10 * i); writer.Append(key, value); } } finally { writer.Close(); } // try splitting the file in a variety of sizes InputFormat <Text, Text> format = new SequenceFileAsTextInputFormat(); for (int i_1 = 0; i_1 < 3; i_1++) { int numSplits = random.Next(MaxLength / (SequenceFile.SyncInterval / 20)) + 1; //LOG.info("splitting: requesting = " + numSplits); InputSplit[] splits = format.GetSplits(job, numSplits); //LOG.info("splitting: got = " + splits.length); // check each split BitSet bits = new BitSet(length); for (int j = 0; j < splits.Length; j++) { RecordReader <Text, Text> reader = format.GetRecordReader(splits[j], job, reporter ); Type readerClass = reader.GetType(); NUnit.Framework.Assert.AreEqual("reader class is SequenceFileAsTextRecordReader." , typeof(SequenceFileAsTextRecordReader), readerClass); Text value = reader.CreateValue(); Text key = reader.CreateKey(); try { int count = 0; while (reader.Next(key, value)) { // if (bits.get(key.get())) { // LOG.info("splits["+j+"]="+splits[j]+" : " + key.get()); // LOG.info("@"+reader.getPos()); // } int keyInt = System.Convert.ToInt32(key.ToString()); NUnit.Framework.Assert.IsFalse("Key in multiple partitions.", bits.Get(keyInt)); bits.Set(keyInt); count++; } } finally { //LOG.info("splits["+j+"]="+splits[j]+" count=" + count); reader.Close(); } } NUnit.Framework.Assert.AreEqual("Some keys in no partition.", length, bits.Cardinality ()); } } }
/// <exception cref="System.Exception"/> public virtual void TestFormat() { Job job = Job.GetInstance(conf); FileSystem fs = FileSystem.GetLocal(conf); Path dir = new Path(Runtime.GetProperty("test.build.data", ".") + "/mapred"); Path file = new Path(dir, "test.seq"); int seed = new Random().Next(); Random random = new Random(seed); fs.Delete(dir, true); FileInputFormat.SetInputPaths(job, dir); // for a variety of lengths for (int length = 0; length < MaxLength; length += random.Next(MaxLength / 10) + 1) { // create a file with length entries SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(IntWritable ), typeof(LongWritable)); try { for (int i = 0; i < length; i++) { IntWritable key = new IntWritable(i); LongWritable value = new LongWritable(10 * i); writer.Append(key, value); } } finally { writer.Close(); } TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(job .GetConfiguration()); // try splitting the file in a variety of sizes InputFormat <Text, Text> format = new SequenceFileAsTextInputFormat(); for (int i_1 = 0; i_1 < 3; i_1++) { // check each split BitSet bits = new BitSet(length); int numSplits = random.Next(MaxLength / (SequenceFile.SyncInterval / 20)) + 1; FileInputFormat.SetMaxInputSplitSize(job, fs.GetFileStatus(file).GetLen() / numSplits ); foreach (InputSplit split in format.GetSplits(job)) { RecordReader <Text, Text> reader = format.CreateRecordReader(split, context); MapContext <Text, Text, Text, Text> mcontext = new MapContextImpl <Text, Text, Text , Text>(job.GetConfiguration(), context.GetTaskAttemptID(), reader, null, null, MapReduceTestUtil.CreateDummyReporter(), split); reader.Initialize(split, mcontext); Type readerClass = reader.GetType(); NUnit.Framework.Assert.AreEqual("reader class is SequenceFileAsTextRecordReader." , typeof(SequenceFileAsTextRecordReader), readerClass); Text key; try { int count = 0; while (reader.NextKeyValue()) { key = reader.GetCurrentKey(); int keyInt = System.Convert.ToInt32(key.ToString()); NUnit.Framework.Assert.IsFalse("Key in multiple partitions.", bits.Get(keyInt)); bits.Set(keyInt); count++; } } finally { reader.Close(); } } NUnit.Framework.Assert.AreEqual("Some keys in no partition.", length, bits.Cardinality ()); } } }
/// <exception cref="System.Exception"/> private static void Launch() { // // Generate distribution of ints. This is the answer key. // Configuration conf = new Configuration(); int countsToGo = counts; int[] dist = new int[range]; for (int i = 0; i < range; i++) { double avgInts = (1.0 * countsToGo) / (range - i); dist[i] = (int)Math.Max(0, Math.Round(avgInts + (Math.Sqrt(avgInts) * r.NextGaussian ()))); countsToGo -= dist[i]; } if (countsToGo > 0) { dist[dist.Length - 1] += countsToGo; } // // Write the answer key to a file. // Path testdir = new Path(TestDir.GetAbsolutePath()); if (!fs.Mkdirs(testdir)) { throw new IOException("Mkdirs failed to create " + testdir.ToString()); } Path randomIns = new Path(testdir, "genins"); if (!fs.Mkdirs(randomIns)) { throw new IOException("Mkdirs failed to create " + randomIns.ToString()); } Path answerkey = new Path(randomIns, "answer.key"); SequenceFile.Writer @out = SequenceFile.CreateWriter(fs, conf, answerkey, typeof( IntWritable), typeof(IntWritable), SequenceFile.CompressionType.None); try { for (int i_1 = 0; i_1 < range; i_1++) { @out.Append(new IntWritable(i_1), new IntWritable(dist[i_1])); } } finally { @out.Close(); } PrintFiles(randomIns, conf); // // Now we need to generate the random numbers according to // the above distribution. // // We create a lot of map tasks, each of which takes at least // one "line" of the distribution. (That is, a certain number // X is to be generated Y number of times.) // // A map task emits Y key/val pairs. The val is X. The key // is a randomly-generated number. // // The reduce task gets its input sorted by key. That is, sorted // in random order. It then emits a single line of text that // for the given values. It does not emit the key. // // Because there's just one reduce task, we emit a single big // file of random numbers. // Path randomOuts = new Path(testdir, "genouts"); fs.Delete(randomOuts, true); Job genJob = Job.GetInstance(conf); FileInputFormat.SetInputPaths(genJob, randomIns); genJob.SetInputFormatClass(typeof(SequenceFileInputFormat)); genJob.SetMapperClass(typeof(TestMapReduce.RandomGenMapper)); FileOutputFormat.SetOutputPath(genJob, randomOuts); genJob.SetOutputKeyClass(typeof(IntWritable)); genJob.SetOutputValueClass(typeof(IntWritable)); genJob.SetReducerClass(typeof(TestMapReduce.RandomGenReducer)); genJob.SetNumReduceTasks(1); genJob.WaitForCompletion(true); PrintFiles(randomOuts, conf); // // Next, we read the big file in and regenerate the // original map. It's split into a number of parts. // (That number is 'intermediateReduces'.) // // We have many map tasks, each of which read at least one // of the output numbers. For each number read in, the // map task emits a key/value pair where the key is the // number and the value is "1". // // We have a single reduce task, which receives its input // sorted by the key emitted above. For each key, there will // be a certain number of "1" values. The reduce task sums // these values to compute how many times the given key was // emitted. // // The reduce task then emits a key/val pair where the key // is the number in question, and the value is the number of // times the key was emitted. This is the same format as the // original answer key (except that numbers emitted zero times // will not appear in the regenerated key.) The answer set // is split into a number of pieces. A final MapReduce job // will merge them. // // There's not really a need to go to 10 reduces here // instead of 1. But we want to test what happens when // you have multiple reduces at once. // int intermediateReduces = 10; Path intermediateOuts = new Path(testdir, "intermediateouts"); fs.Delete(intermediateOuts, true); Job checkJob = Job.GetInstance(conf); FileInputFormat.SetInputPaths(checkJob, randomOuts); checkJob.SetMapperClass(typeof(TestMapReduce.RandomCheckMapper)); FileOutputFormat.SetOutputPath(checkJob, intermediateOuts); checkJob.SetOutputKeyClass(typeof(IntWritable)); checkJob.SetOutputValueClass(typeof(IntWritable)); checkJob.SetOutputFormatClass(typeof(MapFileOutputFormat)); checkJob.SetReducerClass(typeof(TestMapReduce.RandomCheckReducer)); checkJob.SetNumReduceTasks(intermediateReduces); checkJob.WaitForCompletion(true); PrintFiles(intermediateOuts, conf); // // OK, now we take the output from the last job and // merge it down to a single file. The map() and reduce() // functions don't really do anything except reemit tuples. // But by having a single reduce task here, we end up merging // all the files. // Path finalOuts = new Path(testdir, "finalouts"); fs.Delete(finalOuts, true); Job mergeJob = Job.GetInstance(conf); FileInputFormat.SetInputPaths(mergeJob, intermediateOuts); mergeJob.SetInputFormatClass(typeof(SequenceFileInputFormat)); mergeJob.SetMapperClass(typeof(TestMapReduce.MergeMapper)); FileOutputFormat.SetOutputPath(mergeJob, finalOuts); mergeJob.SetOutputKeyClass(typeof(IntWritable)); mergeJob.SetOutputValueClass(typeof(IntWritable)); mergeJob.SetOutputFormatClass(typeof(SequenceFileOutputFormat)); mergeJob.SetReducerClass(typeof(TestMapReduce.MergeReducer)); mergeJob.SetNumReduceTasks(1); mergeJob.WaitForCompletion(true); PrintFiles(finalOuts, conf); // // Finally, we compare the reconstructed answer key with the // original one. Remember, we need to ignore zero-count items // in the original key. // bool success = true; Path recomputedkey = new Path(finalOuts, "part-r-00000"); SequenceFile.Reader @in = new SequenceFile.Reader(fs, recomputedkey, conf); int totalseen = 0; try { IntWritable key = new IntWritable(); IntWritable val = new IntWritable(); for (int i_1 = 0; i_1 < range; i_1++) { if (dist[i_1] == 0) { continue; } if ([email protected](key, val)) { System.Console.Error.WriteLine("Cannot read entry " + i_1); success = false; break; } else { if (!((key.Get() == i_1) && (val.Get() == dist[i_1]))) { System.Console.Error.WriteLine("Mismatch! Pos=" + key.Get() + ", i=" + i_1 + ", val=" + val.Get() + ", dist[i]=" + dist[i_1]); success = false; } totalseen += val.Get(); } } if (success) { if (@in.Next(key, val)) { System.Console.Error.WriteLine("Unnecessary lines in recomputed key!"); success = false; } } } finally { @in.Close(); } int originalTotal = 0; for (int i_2 = 0; i_2 < dist.Length; i_2++) { originalTotal += dist[i_2]; } System.Console.Out.WriteLine("Original sum: " + originalTotal); System.Console.Out.WriteLine("Recomputed sum: " + totalseen); // // Write to "results" whether the test succeeded or not. // Path resultFile = new Path(testdir, "results"); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(fs.Create(resultFile ))); try { bw.Write("Success=" + success + "\n"); System.Console.Out.WriteLine("Success=" + success); } finally { bw.Close(); } NUnit.Framework.Assert.IsTrue("testMapRed failed", success); fs.Delete(testdir, true); }
/// <exception cref="System.Exception"/> public virtual void TestAppendBlockCompression() { GenericTestUtils.AssumeInNativeProfile(); Path file = new Path(RootPath, "testseqappendblockcompr.seq"); fs.Delete(file, true); SequenceFile.Writer.Option compressOption = SequenceFile.Writer.Compression(SequenceFile.CompressionType .Block, new GzipCodec()); SequenceFile.Writer writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer. File(file), SequenceFile.Writer.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass (typeof(string)), compressOption); writer.Append(1L, "one"); writer.Append(2L, "two"); writer.Close(); Verify2Values(file); writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer .KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer .AppendIfExists(true), compressOption); writer.Append(3L, "three"); writer.Append(4L, "four"); writer.Close(); VerifyAll4Values(file); // Verify failure if the compression details are different or not Provided try { writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer .KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer .AppendIfExists(true)); writer.Close(); NUnit.Framework.Assert.Fail("Expected IllegalArgumentException for compression options" ); } catch (ArgumentException) { } // Expected exception. Ignore it // Verify failure if the compression details are different try { SequenceFile.Writer.Option wrongCompressOption = SequenceFile.Writer.Compression( SequenceFile.CompressionType.Record, new GzipCodec()); writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer .KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer .AppendIfExists(true), wrongCompressOption); writer.Close(); NUnit.Framework.Assert.Fail("Expected IllegalArgumentException for compression options" ); } catch (ArgumentException) { } // Expected exception. Ignore it try { SequenceFile.Writer.Option wrongCompressOption = SequenceFile.Writer.Compression( SequenceFile.CompressionType.Block, new DefaultCodec()); writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer .KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer .AppendIfExists(true), wrongCompressOption); writer.Close(); NUnit.Framework.Assert.Fail("Expected IllegalArgumentException for compression options" ); } catch (ArgumentException) { } // Expected exception. Ignore it fs.DeleteOnExit(file); }
/// <exception cref="System.Exception"/> public virtual void TestAppend() { Path file = new Path(RootPath, "testseqappend.seq"); fs.Delete(file, true); Text key1 = new Text("Key1"); Text value1 = new Text("Value1"); Text value2 = new Text("Updated"); SequenceFile.Metadata metadata = new SequenceFile.Metadata(); metadata.Set(key1, value1); SequenceFile.Writer.Option metadataOption = SequenceFile.Writer.Metadata(metadata ); SequenceFile.Writer writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer. File(file), SequenceFile.Writer.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass (typeof(string)), metadataOption); writer.Append(1L, "one"); writer.Append(2L, "two"); writer.Close(); Verify2Values(file); metadata.Set(key1, value2); writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer .KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer .AppendIfExists(true), metadataOption); // Verify the Meta data is not changed Assert.Equal(value1, writer.metadata.Get(key1)); writer.Append(3L, "three"); writer.Append(4L, "four"); writer.Close(); VerifyAll4Values(file); // Verify the Meta data readable after append SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.File (file)); Assert.Equal(value1, reader.GetMetadata().Get(key1)); reader.Close(); // Verify failure if the compression details are different try { SequenceFile.Writer.Option wrongCompressOption = SequenceFile.Writer.Compression( SequenceFile.CompressionType.Record, new GzipCodec()); writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer .KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer .AppendIfExists(true), wrongCompressOption); writer.Close(); NUnit.Framework.Assert.Fail("Expected IllegalArgumentException for compression options" ); } catch (ArgumentException) { } // Expected exception. Ignore it try { SequenceFile.Writer.Option wrongCompressOption = SequenceFile.Writer.Compression( SequenceFile.CompressionType.Block, new DefaultCodec()); writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer .KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer .AppendIfExists(true), wrongCompressOption); writer.Close(); NUnit.Framework.Assert.Fail("Expected IllegalArgumentException for compression options" ); } catch (ArgumentException) { } // Expected exception. Ignore it fs.DeleteOnExit(file); }
/// <summary>Run a map/reduce job for estimating Pi.</summary> /// <returns>the estimated value of Pi</returns> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.TypeLoadException"/> /// <exception cref="System.Exception"/> public static BigDecimal EstimatePi(int numMaps, long numPoints, Path tmpDir, Configuration conf) { Job job = Job.GetInstance(conf); //setup job conf job.SetJobName(typeof(QuasiMonteCarlo).Name); job.SetJarByClass(typeof(QuasiMonteCarlo)); job.SetInputFormatClass(typeof(SequenceFileInputFormat)); job.SetOutputKeyClass(typeof(BooleanWritable)); job.SetOutputValueClass(typeof(LongWritable)); job.SetOutputFormatClass(typeof(SequenceFileOutputFormat)); job.SetMapperClass(typeof(QuasiMonteCarlo.QmcMapper)); job.SetReducerClass(typeof(QuasiMonteCarlo.QmcReducer)); job.SetNumReduceTasks(1); // turn off speculative execution, because DFS doesn't handle // multiple writers to the same file. job.SetSpeculativeExecution(false); //setup input/output directories Path inDir = new Path(tmpDir, "in"); Path outDir = new Path(tmpDir, "out"); FileInputFormat.SetInputPaths(job, inDir); FileOutputFormat.SetOutputPath(job, outDir); FileSystem fs = FileSystem.Get(conf); if (fs.Exists(tmpDir)) { throw new IOException("Tmp directory " + fs.MakeQualified(tmpDir) + " already exists. Please remove it first." ); } if (!fs.Mkdirs(inDir)) { throw new IOException("Cannot create input directory " + inDir); } try { //generate an input file for each map task for (int i = 0; i < numMaps; ++i) { Path file = new Path(inDir, "part" + i); LongWritable offset = new LongWritable(i * numPoints); LongWritable size = new LongWritable(numPoints); SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(LongWritable ), typeof(LongWritable), SequenceFile.CompressionType.None); try { writer.Append(offset, size); } finally { writer.Close(); } System.Console.Out.WriteLine("Wrote input for Map #" + i); } //start a map/reduce job System.Console.Out.WriteLine("Starting Job"); long startTime = Runtime.CurrentTimeMillis(); job.WaitForCompletion(true); double duration = (Runtime.CurrentTimeMillis() - startTime) / 1000.0; System.Console.Out.WriteLine("Job Finished in " + duration + " seconds"); //read outputs Path inFile = new Path(outDir, "reduce-out"); LongWritable numInside = new LongWritable(); LongWritable numOutside = new LongWritable(); SequenceFile.Reader reader = new SequenceFile.Reader(fs, inFile, conf); try { reader.Next(numInside, numOutside); } finally { reader.Close(); } //compute estimated value BigDecimal numTotal = BigDecimal.ValueOf(numMaps).Multiply(BigDecimal.ValueOf(numPoints )); return(BigDecimal.ValueOf(4).SetScale(20).Multiply(BigDecimal.ValueOf(numInside.Get ())).Divide(numTotal, RoundingMode.HalfUp)); } finally { fs.Delete(tmpDir, true); } }