/// <exception cref="System.Exception"/> public virtual void TestBadLength() { int Dlen = 100; DataOutputBuffer dob = new DataOutputBuffer(Dlen + 4); IFileOutputStream ifos = new IFileOutputStream(dob); for (int i = 0; i < Dlen; ++i) { ifos.Write(i); } ifos.Close(); DataInputBuffer dib = new DataInputBuffer(); dib.Reset(dob.GetData(), Dlen + 4); IFileInputStream ifis = new IFileInputStream(dib, 100, new Configuration()); int i_1 = 0; try { while (i_1 < Dlen - 8) { NUnit.Framework.Assert.AreEqual(i_1++, ifis.Read()); } ifis.Close(); } catch (ChecksumException) { NUnit.Framework.Assert.AreEqual("Checksum before close", i_1, Dlen - 8); return; } Fail("Did not detect bad data in checksum"); }
/// <exception cref="System.IO.IOException"/> private void CopyPartitions(Path mapOutputPath, Path indexPath) { FileSystem localFs = FileSystem.GetLocal(jobConf); FileSystem rfs = ((LocalFileSystem)localFs).GetRaw(); FSDataOutputStream rawOutput = rfs.Create(mapOutputPath, true, BufSize); SpillRecord spillRecord = new SpillRecord(numberOfPartitions); IndexRecord indexRecord = new IndexRecord(); for (int i = 0; i < numberOfPartitions; i++) { indexRecord.startOffset = rawOutput.GetPos(); byte[] buffer = outStreams[i].ToByteArray(); IFileOutputStream checksumOutput = new IFileOutputStream(rawOutput); checksumOutput.Write(buffer); // Write checksum. checksumOutput.Finish(); // Write index record indexRecord.rawLength = (long)buffer.Length; indexRecord.partLength = rawOutput.GetPos() - indexRecord.startOffset; spillRecord.PutIndex(indexRecord, i); reporter.Progress(); } rawOutput.Close(); spillRecord.WriteToFile(indexPath, jobConf); }
/// <exception cref="System.Exception"/> public virtual void TestBadIFileStream() { int Dlen = 100; DataOutputBuffer dob = new DataOutputBuffer(Dlen + 4); IFileOutputStream ifos = new IFileOutputStream(dob); for (int i = 0; i < Dlen; ++i) { ifos.Write(i); } ifos.Close(); DataInputBuffer dib = new DataInputBuffer(); byte[] b = dob.GetData(); ++b[17]; dib.Reset(b, Dlen + 4); IFileInputStream ifis = new IFileInputStream(dib, 104, new Configuration()); int i_1 = 0; try { while (i_1 < Dlen) { if (17 == i_1) { NUnit.Framework.Assert.AreEqual(18, ifis.Read()); } else { NUnit.Framework.Assert.AreEqual(i_1, ifis.Read()); } ++i_1; } ifis.Close(); } catch (ChecksumException) { NUnit.Framework.Assert.AreEqual("Unexpected bad checksum", Dlen - 1, i_1); return; } Fail("Did not detect bad data in checksum"); }
/// <exception cref="System.Exception"/> public virtual void TestIFileStream() { int Dlen = 100; DataOutputBuffer dob = new DataOutputBuffer(Dlen + 4); IFileOutputStream ifos = new IFileOutputStream(dob); for (int i = 0; i < Dlen; ++i) { ifos.Write(i); } ifos.Close(); DataInputBuffer dib = new DataInputBuffer(); dib.Reset(dob.GetData(), Dlen + 4); IFileInputStream ifis = new IFileInputStream(dib, 104, new Configuration()); for (int i_1 = 0; i_1 < Dlen; ++i_1) { NUnit.Framework.Assert.AreEqual(i_1, ifis.Read()); } ifis.Close(); }
/// <exception cref="System.IO.IOException"/> public Writer(Configuration conf, FSDataOutputStream @out, Type keyClass, Type valueClass , CompressionCodec codec, Counters.Counter writesCounter, bool ownOutputStream) { this.writtenRecordsCounter = writesCounter; this.checksumOut = new IFileOutputStream(@out); this.rawOut = @out; this.start = this.rawOut.GetPos(); if (codec != null) { this.compressor = CodecPool.GetCompressor(codec); if (this.compressor != null) { this.compressor.Reset(); this.compressedOut = codec.CreateOutputStream(checksumOut, compressor); this.@out = new FSDataOutputStream(this.compressedOut, null); this.compressOutput = true; } else { Log.Warn("Could not obtain compressor from CodecPool"); this.@out = new FSDataOutputStream(checksumOut, null); } } else { this.@out = new FSDataOutputStream(checksumOut, null); } this.keyClass = keyClass; this.valueClass = valueClass; if (keyClass != null) { SerializationFactory serializationFactory = new SerializationFactory(conf); this.keySerializer = serializationFactory.GetSerializer(keyClass); this.keySerializer.Open(buffer); this.valueSerializer = serializationFactory.GetSerializer(valueClass); this.valueSerializer.Open(buffer); } this.ownOutputStream = ownOutputStream; }