Ejemplo n.º 1
0
        public virtual void TestGetPos()
        {
            Path testFile = new Path("/testfile+1");
            // Write a test file.
            FSDataOutputStream @out = hdfs.Create(testFile, true);

            @out.WriteBytes("0123456789");
            @out.Close();
            FSDataInputStream @in = hftpFs.Open(testFile);

            // Test read().
            for (int i = 0; i < 5; ++i)
            {
                NUnit.Framework.Assert.AreEqual(i, @in.GetPos());
                @in.Read();
            }
            // Test read(b, off, len).
            NUnit.Framework.Assert.AreEqual(5, @in.GetPos());
            byte[] buffer = new byte[10];
            NUnit.Framework.Assert.AreEqual(2, @in.Read(buffer, 0, 2));
            NUnit.Framework.Assert.AreEqual(7, @in.GetPos());
            // Test read(b).
            int bytesRead = @in.Read(buffer);

            NUnit.Framework.Assert.AreEqual(7 + bytesRead, @in.GetPos());
            // Test EOF.
            for (int i_1 = 0; i_1 < 100; ++i_1)
            {
                @in.Read();
            }
            NUnit.Framework.Assert.AreEqual(10, @in.GetPos());
            @in.Close();
        }
 public virtual void TestNegativeSeek()
 {
     instream = GetFileSystem().Open(smallSeekFile);
     Assert.Equal(0, instream.GetPos());
     try
     {
         instream.Seek(-1);
         long p = instream.GetPos();
         Log.Warn("Seek to -1 returned a position of " + p);
         int result = instream.Read();
         NUnit.Framework.Assert.Fail("expected an exception, got data " + result + " at a position of "
                                     + p);
     }
     catch (EOFException e)
     {
         //bad seek -expected
         HandleExpectedException(e);
     }
     catch (IOException e)
     {
         //bad seek -expected, but not as preferred as an EOFException
         HandleRelaxedException("a negative seek", "EOFException", e);
     }
     Assert.Equal(0, instream.GetPos());
 }
        public virtual void TestSeekBigFile()
        {
            Describe("Seek round a large file and verify the bytes are what is expected");
            Path testSeekFile = Path("bigseekfile.txt");

            byte[] block = ContractTestUtils.Dataset(65536, 0, 255);
            ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block);
            instream = GetFileSystem().Open(testSeekFile);
            Assert.Equal(0, instream.GetPos());
            //expect that seek to 0 works
            instream.Seek(0);
            int result = instream.Read();

            Assert.Equal(0, result);
            Assert.Equal(1, instream.Read());
            Assert.Equal(2, instream.Read());
            //do seek 32KB ahead
            instream.Seek(32768);
            Assert.Equal("@32768", block[32768], unchecked ((byte)instream.
                                                            Read()));
            instream.Seek(40000);
            Assert.Equal("@40000", block[40000], unchecked ((byte)instream.
                                                            Read()));
            instream.Seek(8191);
            Assert.Equal("@8191", block[8191], unchecked ((byte)instream.Read
                                                              ()));
            instream.Seek(0);
            Assert.Equal("@0", 0, unchecked ((byte)instream.Read()));
        }
Ejemplo n.º 4
0
        /// <exception cref="System.IO.IOException"/>
        private long DumpFromOffset(PathData item, long offset)
        {
            long fileSize = item.RefreshStatus().GetLen();

            if (offset > fileSize)
            {
                return(fileSize);
            }
            // treat a negative offset as relative to end of the file, floor of 0
            if (offset < 0)
            {
                offset = Math.Max(fileSize + offset, 0);
            }
            FSDataInputStream @in = item.fs.Open(item.path);

            try
            {
                @in.Seek(offset);
                // use conf so the system configured io block size is used
                IOUtils.CopyBytes(@in, System.Console.Out, GetConf(), false);
                offset = @in.GetPos();
            }
            finally
            {
                @in.Close();
            }
            return(offset);
        }
Ejemplo n.º 5
0
        /// <exception cref="System.Exception"/>
        private void ReadTestFile(string testFileName)
        {
            Path filePath             = new Path(testFileName);
            FSDataInputStream istream = dfs.Open(filePath, 10240);
            ByteBuffer        buf     = ByteBuffer.Allocate(10240);
            int count = 0;

            try
            {
                while (istream.Read(buf) > 0)
                {
                    count += 1;
                    buf.Clear();
                    istream.Seek(istream.GetPos() + 5);
                }
            }
            catch (IOException)
            {
            }
            finally
            {
                // Ignore this it's probably a seek after eof.
                istream.Close();
            }
        }
        public virtual void TestSeekFile()
        {
            Describe("basic seek operations");
            instream = GetFileSystem().Open(smallSeekFile);
            Assert.Equal(0, instream.GetPos());
            //expect that seek to 0 works
            instream.Seek(0);
            int result = instream.Read();

            Assert.Equal(0, result);
            Assert.Equal(1, instream.Read());
            Assert.Equal(2, instream.GetPos());
            Assert.Equal(2, instream.Read());
            Assert.Equal(3, instream.GetPos());
            instream.Seek(128);
            Assert.Equal(128, instream.GetPos());
            Assert.Equal(128, instream.Read());
            instream.Seek(63);
            Assert.Equal(63, instream.Read());
        }
        public virtual void TestBlockReadZeroByteFile()
        {
            Describe("do a block read on a 0 byte file");
            instream = GetFileSystem().Open(zeroByteFile);
            Assert.Equal(0, instream.GetPos());
            //expect that seek to 0 works
            byte[] buffer = new byte[1];
            int    result = instream.Read(buffer, 0, 1);

            AssertMinusOne("block read zero byte file", result);
        }
 public virtual void TestSeekAndReadPastEndOfFile()
 {
     Describe("verify that reading past the last bytes in the file returns -1");
     instream = GetFileSystem().Open(smallSeekFile);
     Assert.Equal(0, instream.GetPos());
     //expect that seek to 0 works
     //go just before the end
     instream.Seek(TestFileLen - 2);
     Assert.True("Premature EOF", instream.Read() != -1);
     Assert.True("Premature EOF", instream.Read() != -1);
     AssertMinusOne("read past end of file", instream.Read());
 }
        public virtual void TestOpenReadZeroByteFile()
        {
            Describe("create & read a 0 byte file");
            Path path = Path("zero.txt");

            ContractTestUtils.Touch(GetFileSystem(), path);
            instream = GetFileSystem().Open(path);
            Assert.Equal(0, instream.GetPos());
            //expect initial read to fail
            int result = instream.Read();

            AssertMinusOne("initial byte read", result);
        }
Ejemplo n.º 10
0
            /// <exception cref="System.IO.IOException"/>
            protected internal override int ReadChunk(long pos, byte[] buf, int offset, int len
                                                      , byte[] checksum)
            {
                bool eof = false;

                if (NeedChecksum())
                {
                    System.Diagnostics.Debug.Assert(checksum != null);
                    // we have a checksum buffer
                    System.Diagnostics.Debug.Assert(checksum.Length % ChecksumSize == 0);
                    // it is sane length
                    System.Diagnostics.Debug.Assert(len >= bytesPerSum);
                    // we must read at least one chunk
                    int checksumsToRead = Math.Min(len / bytesPerSum, checksum.Length / ChecksumSize);
                    // number of checksums based on len to read
                    // size of checksum buffer
                    long checksumPos = GetChecksumFilePos(pos);
                    if (checksumPos != sums.GetPos())
                    {
                        sums.Seek(checksumPos);
                    }
                    int sumLenRead = sums.Read(checksum, 0, ChecksumSize * checksumsToRead);
                    if (sumLenRead >= 0 && sumLenRead % ChecksumSize != 0)
                    {
                        throw new ChecksumException("Checksum file not a length multiple of checksum size "
                                                    + "in " + file + " at " + pos + " checksumpos: " + checksumPos + " sumLenread: "
                                                    + sumLenRead, pos);
                    }
                    if (sumLenRead <= 0)
                    {
                        // we're at the end of the file
                        eof = true;
                    }
                    else
                    {
                        // Adjust amount of data to read based on how many checksum chunks we read
                        len = Math.Min(len, bytesPerSum * (sumLenRead / ChecksumSize));
                    }
                }
                if (pos != datas.GetPos())
                {
                    datas.Seek(pos);
                }
                int nread = ReadFully(datas, buf, offset, len);

                if (eof && nread > 0)
                {
                    throw new ChecksumException("Checksum error: " + file + " at " + pos, pos);
                }
                return(nread);
            }
Ejemplo n.º 11
0
        public virtual void TestPositionedBulkReadDoesntChangePosition()
        {
            Describe("verify that a positioned read does not change the getPos() value");
            Path testSeekFile = Path("bigseekfile.txt");

            byte[] block = ContractTestUtils.Dataset(65536, 0, 255);
            ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block);
            instream = GetFileSystem().Open(testSeekFile);
            instream.Seek(39999);
            Assert.True(-1 != instream.Read());
            Assert.Equal(40000, instream.GetPos());
            byte[] readBuffer = new byte[256];
            instream.Read(128, readBuffer, 0, readBuffer.Length);
            //have gone back
            Assert.Equal(40000, instream.GetPos());
            //content is the same too
            Assert.Equal("@40000", block[40000], unchecked ((byte)instream.
                                                            Read()));
            //now verify the picked up data
            for (int i = 0; i < 256; i++)
            {
                Assert.Equal("@" + i, block[i + 128], readBuffer[i]);
            }
        }
Ejemplo n.º 12
0
        public virtual void TestSeekZeroByteFile()
        {
            Describe("seek and read a 0 byte file");
            instream = GetFileSystem().Open(zeroByteFile);
            Assert.Equal(0, instream.GetPos());
            //expect initial read to fai;
            int result = instream.Read();

            AssertMinusOne("initial byte read", result);
            byte[] buffer = new byte[1];
            //expect that seek to 0 works
            instream.Seek(0);
            //reread, expect same exception
            result = instream.Read();
            AssertMinusOne("post-seek byte read", result);
            result = instream.Read(buffer, 0, 1);
            AssertMinusOne("post-seek buffer read", result);
        }
Ejemplo n.º 13
0
        public virtual void TestRamDiskEvictionWithShortCircuitReadHandle()
        {
            StartUpCluster(ReplFactor, new StorageType[] { StorageType.RamDisk, StorageType.Default }, (6 * BlockSize - 1), true);
            // 5 replica + delta, SCR.
            string MethodName = GenericTestUtils.GetMethodName();
            Path   path1      = new Path("/" + MethodName + ".01.dat");
            Path   path2      = new Path("/" + MethodName + ".02.dat");
            int    Seed       = unchecked ((int)(0xFADED));

            MakeRandomTestFile(path1, BlockSize, true, Seed);
            EnsureFileReplicasOnStorageType(path1, StorageType.RamDisk);
            // Sleep for a short time to allow the lazy writer thread to do its job.
            // However the block replica should not be evicted from RAM_DISK yet.
            Sharpen.Thread.Sleep(3 * LazyWriterIntervalSec * 1000);
            // No eviction should happen as the free ratio is below the threshold
            FSDataInputStream fis = fs.Open(path1);

            try
            {
                // Keep and open read handle to path1 while creating path2
                byte[] buf = new byte[BufferLength];
                fis.Read(0, buf, 0, BufferLength);
                // Create the 2nd file that will trigger RAM_DISK eviction.
                MakeTestFile(path2, BlockSize * 2, true);
                EnsureFileReplicasOnStorageType(path2, StorageType.RamDisk);
                // Ensure path1 is still readable from the open SCR handle.
                fis.Read(fis.GetPos(), buf, 0, BufferLength);
                HdfsDataInputStream dfsis = (HdfsDataInputStream)fis;
                NUnit.Framework.Assert.AreEqual(2 * BufferLength, dfsis.GetReadStatistics().GetTotalBytesRead
                                                    ());
                NUnit.Framework.Assert.AreEqual(2 * BufferLength, dfsis.GetReadStatistics().GetTotalShortCircuitBytesRead
                                                    ());
            }
            finally
            {
                IOUtils.CloseQuietly(fis);
            }
            // After the open handle is closed, path1 should be evicted to DISK.
            TriggerBlockReport();
            EnsureFileReplicasOnStorageType(path1, StorageType.Default);
        }
Ejemplo n.º 14
0
        /// <summary>
        /// Test (expected to throw IOE) for <code>FSDataInpuStream#seek</code>
        /// when the position argument is larger than the file size.
        /// </summary>
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestSeekPastFileSize()
        {
            Configuration  conf    = new HdfsConfiguration();
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
            FileSystem     fs      = cluster.GetFileSystem();

            try
            {
                Path seekFile = new Path("seekboundaries.dat");
                DFSTestUtil.CreateFile(fs, seekFile, Onemb, fs.GetDefaultReplication(seekFile), seed
                                       );
                FSDataInputStream stream = fs.Open(seekFile);
                // Perform "safe seek" (expected to pass)
                stream.Seek(65536);
                NUnit.Framework.Assert.AreEqual(65536, stream.GetPos());
                // expect IOE for this call
                stream.Seek(Onemb + Onemb + Onemb);
            }
            finally
            {
                fs.Close();
                cluster.Shutdown();
            }
        }
Ejemplo n.º 15
0
        /// <summary>read chunks into buffer repeatedly until total of VisibleLen byte are read.
        ///     </summary>
        /// <remarks>
        /// read chunks into buffer repeatedly until total of VisibleLen byte are read.
        /// Return total number of bytes read
        /// </remarks>
        /// <exception cref="System.IO.IOException"/>
        private long ReadUntilEnd(FSDataInputStream @in, byte[] buffer, long size, string
                                  fname, long pos, long visibleLen, bool positionReadOption)
        {
            if (pos >= visibleLen || visibleLen <= 0)
            {
                return(0);
            }
            int  chunkNumber         = 0;
            long totalByteRead       = 0;
            long currentPosition     = pos;
            int  byteRead            = 0;
            long byteLeftToRead      = visibleLen - pos;
            int  byteToReadThisRound = 0;

            if (!positionReadOption)
            {
                @in.Seek(pos);
                currentPosition = @in.GetPos();
            }
            if (verboseOption)
            {
                Log.Info("reader begin: position: " + pos + " ; currentOffset = " + currentPosition
                         + " ; bufferSize =" + buffer.Length + " ; Filename = " + fname);
            }
            try
            {
                while (byteLeftToRead > 0 && currentPosition < visibleLen)
                {
                    byteToReadThisRound = (int)(byteLeftToRead >= buffer.Length ? buffer.Length : byteLeftToRead
                                                );
                    if (positionReadOption)
                    {
                        byteRead = @in.Read(currentPosition, buffer, 0, byteToReadThisRound);
                    }
                    else
                    {
                        byteRead = @in.Read(buffer, 0, byteToReadThisRound);
                    }
                    if (byteRead <= 0)
                    {
                        break;
                    }
                    chunkNumber++;
                    totalByteRead   += byteRead;
                    currentPosition += byteRead;
                    byteLeftToRead  -= byteRead;
                    if (verboseOption)
                    {
                        Log.Info("reader: Number of byte read: " + byteRead + " ; totalByteRead = " + totalByteRead
                                 + " ; currentPosition=" + currentPosition + " ; chunkNumber =" + chunkNumber +
                                 "; File name = " + fname);
                    }
                }
            }
            catch (IOException e)
            {
                throw new IOException("#### Exception caught in readUntilEnd: reader  currentOffset = "
                                      + currentPosition + " ; totalByteRead =" + totalByteRead + " ; latest byteRead = "
                                      + byteRead + "; visibleLen= " + visibleLen + " ; bufferLen = " + buffer.Length
                                      + " ; Filename = " + fname, e);
            }
            if (verboseOption)
            {
                Log.Info("reader end:   position: " + pos + " ; currentOffset = " + currentPosition
                         + " ; totalByteRead =" + totalByteRead + " ; Filename = " + fname);
            }
            return(totalByteRead);
        }
Ejemplo n.º 16
0
        /* test read and getPos */
        /// <exception cref="System.Exception"/>
        private void CheckReadAndGetPos()
        {
            actual = new byte[FileSize];
            // test reads that do not cross checksum boundary
            stm.Seek(0);
            int offset;

            for (offset = 0; offset < BlockSize + BytesPerSum; offset += BytesPerSum)
            {
                NUnit.Framework.Assert.AreEqual(stm.GetPos(), offset);
                stm.ReadFully(actual, offset, BytesPerSum);
            }
            stm.ReadFully(actual, offset, FileSize - BlockSize - BytesPerSum);
            NUnit.Framework.Assert.AreEqual(stm.GetPos(), FileSize);
            CheckAndEraseData(actual, 0, expected, "Read Sanity Test");
            // test reads that cross checksum boundary
            stm.Seek(0L);
            NUnit.Framework.Assert.AreEqual(stm.GetPos(), 0L);
            stm.ReadFully(actual, 0, HalfChunkSize);
            NUnit.Framework.Assert.AreEqual(stm.GetPos(), HalfChunkSize);
            stm.ReadFully(actual, HalfChunkSize, BlockSize - HalfChunkSize);
            NUnit.Framework.Assert.AreEqual(stm.GetPos(), BlockSize);
            stm.ReadFully(actual, BlockSize, BytesPerSum + HalfChunkSize);
            NUnit.Framework.Assert.AreEqual(stm.GetPos(), BlockSize + BytesPerSum + HalfChunkSize
                                            );
            stm.ReadFully(actual, 2 * BlockSize - HalfChunkSize, FileSize - (2 * BlockSize -
                                                                             HalfChunkSize));
            NUnit.Framework.Assert.AreEqual(stm.GetPos(), FileSize);
            CheckAndEraseData(actual, 0, expected, "Read Sanity Test");
            // test read that cross block boundary
            stm.Seek(0L);
            stm.ReadFully(actual, 0, BytesPerSum + HalfChunkSize);
            NUnit.Framework.Assert.AreEqual(stm.GetPos(), BytesPerSum + HalfChunkSize);
            stm.ReadFully(actual, BytesPerSum + HalfChunkSize, BytesPerSum);
            NUnit.Framework.Assert.AreEqual(stm.GetPos(), BlockSize + HalfChunkSize);
            stm.ReadFully(actual, BlockSize + HalfChunkSize, FileSize - BlockSize - HalfChunkSize
                          );
            NUnit.Framework.Assert.AreEqual(stm.GetPos(), FileSize);
            CheckAndEraseData(actual, 0, expected, "Read Sanity Test");
        }
Ejemplo n.º 17
0
        public virtual void TestSeekReadClosedFile()
        {
            bool supportsSeekOnClosedFiles = IsSupported(SupportsSeekOnClosedFile);

            instream = GetFileSystem().Open(smallSeekFile);
            GetLog().Debug("Stream is of type " + instream.GetType().GetCanonicalName());
            instream.Close();
            try
            {
                instream.Seek(0);
                if (!supportsSeekOnClosedFiles)
                {
                    NUnit.Framework.Assert.Fail("seek succeeded on a closed stream");
                }
            }
            catch (IOException)
            {
            }
            //expected a closed file
            try
            {
                int data = instream.Available();
                NUnit.Framework.Assert.Fail("read() succeeded on a closed stream, got " + data);
            }
            catch (IOException)
            {
            }
            //expected a closed file
            try
            {
                int data = instream.Read();
                NUnit.Framework.Assert.Fail("read() succeeded on a closed stream, got " + data);
            }
            catch (IOException)
            {
            }
            //expected a closed file
            try
            {
                byte[] buffer = new byte[1];
                int    result = instream.Read(buffer, 0, 1);
                NUnit.Framework.Assert.Fail("read(buffer, 0, 1) succeeded on a closed stream, got "
                                            + result);
            }
            catch (IOException)
            {
            }
            //expected a closed file
            //what position does a closed file have?
            try
            {
                long offset = instream.GetPos();
            }
            catch (IOException)
            {
            }
            // its valid to raise error here; but the test is applied to make
            // sure there's no other exception like an NPE.
            //and close again
            instream.Close();
        }