public virtual void TestGetPos() { Path testFile = new Path("/testfile+1"); // Write a test file. FSDataOutputStream @out = hdfs.Create(testFile, true); @out.WriteBytes("0123456789"); @out.Close(); FSDataInputStream @in = hftpFs.Open(testFile); // Test read(). for (int i = 0; i < 5; ++i) { NUnit.Framework.Assert.AreEqual(i, @in.GetPos()); @in.Read(); } // Test read(b, off, len). NUnit.Framework.Assert.AreEqual(5, @in.GetPos()); byte[] buffer = new byte[10]; NUnit.Framework.Assert.AreEqual(2, @in.Read(buffer, 0, 2)); NUnit.Framework.Assert.AreEqual(7, @in.GetPos()); // Test read(b). int bytesRead = @in.Read(buffer); NUnit.Framework.Assert.AreEqual(7 + bytesRead, @in.GetPos()); // Test EOF. for (int i_1 = 0; i_1 < 100; ++i_1) { @in.Read(); } NUnit.Framework.Assert.AreEqual(10, @in.GetPos()); @in.Close(); }
public virtual void TestNegativeSeek() { instream = GetFileSystem().Open(smallSeekFile); Assert.Equal(0, instream.GetPos()); try { instream.Seek(-1); long p = instream.GetPos(); Log.Warn("Seek to -1 returned a position of " + p); int result = instream.Read(); NUnit.Framework.Assert.Fail("expected an exception, got data " + result + " at a position of " + p); } catch (EOFException e) { //bad seek -expected HandleExpectedException(e); } catch (IOException e) { //bad seek -expected, but not as preferred as an EOFException HandleRelaxedException("a negative seek", "EOFException", e); } Assert.Equal(0, instream.GetPos()); }
public virtual void TestSeekBigFile() { Describe("Seek round a large file and verify the bytes are what is expected"); Path testSeekFile = Path("bigseekfile.txt"); byte[] block = ContractTestUtils.Dataset(65536, 0, 255); ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block); instream = GetFileSystem().Open(testSeekFile); Assert.Equal(0, instream.GetPos()); //expect that seek to 0 works instream.Seek(0); int result = instream.Read(); Assert.Equal(0, result); Assert.Equal(1, instream.Read()); Assert.Equal(2, instream.Read()); //do seek 32KB ahead instream.Seek(32768); Assert.Equal("@32768", block[32768], unchecked ((byte)instream. Read())); instream.Seek(40000); Assert.Equal("@40000", block[40000], unchecked ((byte)instream. Read())); instream.Seek(8191); Assert.Equal("@8191", block[8191], unchecked ((byte)instream.Read ())); instream.Seek(0); Assert.Equal("@0", 0, unchecked ((byte)instream.Read())); }
/// <exception cref="System.IO.IOException"/> private long DumpFromOffset(PathData item, long offset) { long fileSize = item.RefreshStatus().GetLen(); if (offset > fileSize) { return(fileSize); } // treat a negative offset as relative to end of the file, floor of 0 if (offset < 0) { offset = Math.Max(fileSize + offset, 0); } FSDataInputStream @in = item.fs.Open(item.path); try { @in.Seek(offset); // use conf so the system configured io block size is used IOUtils.CopyBytes(@in, System.Console.Out, GetConf(), false); offset = @in.GetPos(); } finally { @in.Close(); } return(offset); }
/// <exception cref="System.Exception"/> private void ReadTestFile(string testFileName) { Path filePath = new Path(testFileName); FSDataInputStream istream = dfs.Open(filePath, 10240); ByteBuffer buf = ByteBuffer.Allocate(10240); int count = 0; try { while (istream.Read(buf) > 0) { count += 1; buf.Clear(); istream.Seek(istream.GetPos() + 5); } } catch (IOException) { } finally { // Ignore this it's probably a seek after eof. istream.Close(); } }
public virtual void TestSeekFile() { Describe("basic seek operations"); instream = GetFileSystem().Open(smallSeekFile); Assert.Equal(0, instream.GetPos()); //expect that seek to 0 works instream.Seek(0); int result = instream.Read(); Assert.Equal(0, result); Assert.Equal(1, instream.Read()); Assert.Equal(2, instream.GetPos()); Assert.Equal(2, instream.Read()); Assert.Equal(3, instream.GetPos()); instream.Seek(128); Assert.Equal(128, instream.GetPos()); Assert.Equal(128, instream.Read()); instream.Seek(63); Assert.Equal(63, instream.Read()); }
public virtual void TestBlockReadZeroByteFile() { Describe("do a block read on a 0 byte file"); instream = GetFileSystem().Open(zeroByteFile); Assert.Equal(0, instream.GetPos()); //expect that seek to 0 works byte[] buffer = new byte[1]; int result = instream.Read(buffer, 0, 1); AssertMinusOne("block read zero byte file", result); }
public virtual void TestSeekAndReadPastEndOfFile() { Describe("verify that reading past the last bytes in the file returns -1"); instream = GetFileSystem().Open(smallSeekFile); Assert.Equal(0, instream.GetPos()); //expect that seek to 0 works //go just before the end instream.Seek(TestFileLen - 2); Assert.True("Premature EOF", instream.Read() != -1); Assert.True("Premature EOF", instream.Read() != -1); AssertMinusOne("read past end of file", instream.Read()); }
public virtual void TestOpenReadZeroByteFile() { Describe("create & read a 0 byte file"); Path path = Path("zero.txt"); ContractTestUtils.Touch(GetFileSystem(), path); instream = GetFileSystem().Open(path); Assert.Equal(0, instream.GetPos()); //expect initial read to fail int result = instream.Read(); AssertMinusOne("initial byte read", result); }
/// <exception cref="System.IO.IOException"/> protected internal override int ReadChunk(long pos, byte[] buf, int offset, int len , byte[] checksum) { bool eof = false; if (NeedChecksum()) { System.Diagnostics.Debug.Assert(checksum != null); // we have a checksum buffer System.Diagnostics.Debug.Assert(checksum.Length % ChecksumSize == 0); // it is sane length System.Diagnostics.Debug.Assert(len >= bytesPerSum); // we must read at least one chunk int checksumsToRead = Math.Min(len / bytesPerSum, checksum.Length / ChecksumSize); // number of checksums based on len to read // size of checksum buffer long checksumPos = GetChecksumFilePos(pos); if (checksumPos != sums.GetPos()) { sums.Seek(checksumPos); } int sumLenRead = sums.Read(checksum, 0, ChecksumSize * checksumsToRead); if (sumLenRead >= 0 && sumLenRead % ChecksumSize != 0) { throw new ChecksumException("Checksum file not a length multiple of checksum size " + "in " + file + " at " + pos + " checksumpos: " + checksumPos + " sumLenread: " + sumLenRead, pos); } if (sumLenRead <= 0) { // we're at the end of the file eof = true; } else { // Adjust amount of data to read based on how many checksum chunks we read len = Math.Min(len, bytesPerSum * (sumLenRead / ChecksumSize)); } } if (pos != datas.GetPos()) { datas.Seek(pos); } int nread = ReadFully(datas, buf, offset, len); if (eof && nread > 0) { throw new ChecksumException("Checksum error: " + file + " at " + pos, pos); } return(nread); }
public virtual void TestPositionedBulkReadDoesntChangePosition() { Describe("verify that a positioned read does not change the getPos() value"); Path testSeekFile = Path("bigseekfile.txt"); byte[] block = ContractTestUtils.Dataset(65536, 0, 255); ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block); instream = GetFileSystem().Open(testSeekFile); instream.Seek(39999); Assert.True(-1 != instream.Read()); Assert.Equal(40000, instream.GetPos()); byte[] readBuffer = new byte[256]; instream.Read(128, readBuffer, 0, readBuffer.Length); //have gone back Assert.Equal(40000, instream.GetPos()); //content is the same too Assert.Equal("@40000", block[40000], unchecked ((byte)instream. Read())); //now verify the picked up data for (int i = 0; i < 256; i++) { Assert.Equal("@" + i, block[i + 128], readBuffer[i]); } }
public virtual void TestSeekZeroByteFile() { Describe("seek and read a 0 byte file"); instream = GetFileSystem().Open(zeroByteFile); Assert.Equal(0, instream.GetPos()); //expect initial read to fai; int result = instream.Read(); AssertMinusOne("initial byte read", result); byte[] buffer = new byte[1]; //expect that seek to 0 works instream.Seek(0); //reread, expect same exception result = instream.Read(); AssertMinusOne("post-seek byte read", result); result = instream.Read(buffer, 0, 1); AssertMinusOne("post-seek buffer read", result); }
public virtual void TestRamDiskEvictionWithShortCircuitReadHandle() { StartUpCluster(ReplFactor, new StorageType[] { StorageType.RamDisk, StorageType.Default }, (6 * BlockSize - 1), true); // 5 replica + delta, SCR. string MethodName = GenericTestUtils.GetMethodName(); Path path1 = new Path("/" + MethodName + ".01.dat"); Path path2 = new Path("/" + MethodName + ".02.dat"); int Seed = unchecked ((int)(0xFADED)); MakeRandomTestFile(path1, BlockSize, true, Seed); EnsureFileReplicasOnStorageType(path1, StorageType.RamDisk); // Sleep for a short time to allow the lazy writer thread to do its job. // However the block replica should not be evicted from RAM_DISK yet. Sharpen.Thread.Sleep(3 * LazyWriterIntervalSec * 1000); // No eviction should happen as the free ratio is below the threshold FSDataInputStream fis = fs.Open(path1); try { // Keep and open read handle to path1 while creating path2 byte[] buf = new byte[BufferLength]; fis.Read(0, buf, 0, BufferLength); // Create the 2nd file that will trigger RAM_DISK eviction. MakeTestFile(path2, BlockSize * 2, true); EnsureFileReplicasOnStorageType(path2, StorageType.RamDisk); // Ensure path1 is still readable from the open SCR handle. fis.Read(fis.GetPos(), buf, 0, BufferLength); HdfsDataInputStream dfsis = (HdfsDataInputStream)fis; NUnit.Framework.Assert.AreEqual(2 * BufferLength, dfsis.GetReadStatistics().GetTotalBytesRead ()); NUnit.Framework.Assert.AreEqual(2 * BufferLength, dfsis.GetReadStatistics().GetTotalShortCircuitBytesRead ()); } finally { IOUtils.CloseQuietly(fis); } // After the open handle is closed, path1 should be evicted to DISK. TriggerBlockReport(); EnsureFileReplicasOnStorageType(path1, StorageType.Default); }
/// <summary> /// Test (expected to throw IOE) for <code>FSDataInpuStream#seek</code> /// when the position argument is larger than the file size. /// </summary> /// <exception cref="System.IO.IOException"/> public virtual void TestSeekPastFileSize() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); try { Path seekFile = new Path("seekboundaries.dat"); DFSTestUtil.CreateFile(fs, seekFile, Onemb, fs.GetDefaultReplication(seekFile), seed ); FSDataInputStream stream = fs.Open(seekFile); // Perform "safe seek" (expected to pass) stream.Seek(65536); NUnit.Framework.Assert.AreEqual(65536, stream.GetPos()); // expect IOE for this call stream.Seek(Onemb + Onemb + Onemb); } finally { fs.Close(); cluster.Shutdown(); } }
/// <summary>read chunks into buffer repeatedly until total of VisibleLen byte are read. /// </summary> /// <remarks> /// read chunks into buffer repeatedly until total of VisibleLen byte are read. /// Return total number of bytes read /// </remarks> /// <exception cref="System.IO.IOException"/> private long ReadUntilEnd(FSDataInputStream @in, byte[] buffer, long size, string fname, long pos, long visibleLen, bool positionReadOption) { if (pos >= visibleLen || visibleLen <= 0) { return(0); } int chunkNumber = 0; long totalByteRead = 0; long currentPosition = pos; int byteRead = 0; long byteLeftToRead = visibleLen - pos; int byteToReadThisRound = 0; if (!positionReadOption) { @in.Seek(pos); currentPosition = @in.GetPos(); } if (verboseOption) { Log.Info("reader begin: position: " + pos + " ; currentOffset = " + currentPosition + " ; bufferSize =" + buffer.Length + " ; Filename = " + fname); } try { while (byteLeftToRead > 0 && currentPosition < visibleLen) { byteToReadThisRound = (int)(byteLeftToRead >= buffer.Length ? buffer.Length : byteLeftToRead ); if (positionReadOption) { byteRead = @in.Read(currentPosition, buffer, 0, byteToReadThisRound); } else { byteRead = @in.Read(buffer, 0, byteToReadThisRound); } if (byteRead <= 0) { break; } chunkNumber++; totalByteRead += byteRead; currentPosition += byteRead; byteLeftToRead -= byteRead; if (verboseOption) { Log.Info("reader: Number of byte read: " + byteRead + " ; totalByteRead = " + totalByteRead + " ; currentPosition=" + currentPosition + " ; chunkNumber =" + chunkNumber + "; File name = " + fname); } } } catch (IOException e) { throw new IOException("#### Exception caught in readUntilEnd: reader currentOffset = " + currentPosition + " ; totalByteRead =" + totalByteRead + " ; latest byteRead = " + byteRead + "; visibleLen= " + visibleLen + " ; bufferLen = " + buffer.Length + " ; Filename = " + fname, e); } if (verboseOption) { Log.Info("reader end: position: " + pos + " ; currentOffset = " + currentPosition + " ; totalByteRead =" + totalByteRead + " ; Filename = " + fname); } return(totalByteRead); }
/* test read and getPos */ /// <exception cref="System.Exception"/> private void CheckReadAndGetPos() { actual = new byte[FileSize]; // test reads that do not cross checksum boundary stm.Seek(0); int offset; for (offset = 0; offset < BlockSize + BytesPerSum; offset += BytesPerSum) { NUnit.Framework.Assert.AreEqual(stm.GetPos(), offset); stm.ReadFully(actual, offset, BytesPerSum); } stm.ReadFully(actual, offset, FileSize - BlockSize - BytesPerSum); NUnit.Framework.Assert.AreEqual(stm.GetPos(), FileSize); CheckAndEraseData(actual, 0, expected, "Read Sanity Test"); // test reads that cross checksum boundary stm.Seek(0L); NUnit.Framework.Assert.AreEqual(stm.GetPos(), 0L); stm.ReadFully(actual, 0, HalfChunkSize); NUnit.Framework.Assert.AreEqual(stm.GetPos(), HalfChunkSize); stm.ReadFully(actual, HalfChunkSize, BlockSize - HalfChunkSize); NUnit.Framework.Assert.AreEqual(stm.GetPos(), BlockSize); stm.ReadFully(actual, BlockSize, BytesPerSum + HalfChunkSize); NUnit.Framework.Assert.AreEqual(stm.GetPos(), BlockSize + BytesPerSum + HalfChunkSize ); stm.ReadFully(actual, 2 * BlockSize - HalfChunkSize, FileSize - (2 * BlockSize - HalfChunkSize)); NUnit.Framework.Assert.AreEqual(stm.GetPos(), FileSize); CheckAndEraseData(actual, 0, expected, "Read Sanity Test"); // test read that cross block boundary stm.Seek(0L); stm.ReadFully(actual, 0, BytesPerSum + HalfChunkSize); NUnit.Framework.Assert.AreEqual(stm.GetPos(), BytesPerSum + HalfChunkSize); stm.ReadFully(actual, BytesPerSum + HalfChunkSize, BytesPerSum); NUnit.Framework.Assert.AreEqual(stm.GetPos(), BlockSize + HalfChunkSize); stm.ReadFully(actual, BlockSize + HalfChunkSize, FileSize - BlockSize - HalfChunkSize ); NUnit.Framework.Assert.AreEqual(stm.GetPos(), FileSize); CheckAndEraseData(actual, 0, expected, "Read Sanity Test"); }
public virtual void TestSeekReadClosedFile() { bool supportsSeekOnClosedFiles = IsSupported(SupportsSeekOnClosedFile); instream = GetFileSystem().Open(smallSeekFile); GetLog().Debug("Stream is of type " + instream.GetType().GetCanonicalName()); instream.Close(); try { instream.Seek(0); if (!supportsSeekOnClosedFiles) { NUnit.Framework.Assert.Fail("seek succeeded on a closed stream"); } } catch (IOException) { } //expected a closed file try { int data = instream.Available(); NUnit.Framework.Assert.Fail("read() succeeded on a closed stream, got " + data); } catch (IOException) { } //expected a closed file try { int data = instream.Read(); NUnit.Framework.Assert.Fail("read() succeeded on a closed stream, got " + data); } catch (IOException) { } //expected a closed file try { byte[] buffer = new byte[1]; int result = instream.Read(buffer, 0, 1); NUnit.Framework.Assert.Fail("read(buffer, 0, 1) succeeded on a closed stream, got " + result); } catch (IOException) { } //expected a closed file //what position does a closed file have? try { long offset = instream.GetPos(); } catch (IOException) { } // its valid to raise error here; but the test is applied to make // sure there's no other exception like an NPE. //and close again instream.Close(); }