/// <summary> /// Check the file content, reading as user /// <paramref name="readingUser"/> /// /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal static void CheckFileContentDirect(URI uri, Path name, byte[] expected, int readOffset, string readingUser, Configuration conf, bool legacyShortCircuitFails ) { // Ensure short circuit is enabled DistributedFileSystem fs = GetFileSystem(readingUser, uri, conf); ClientContext clientContext = ClientContext.GetFromConf(conf); if (legacyShortCircuitFails) { NUnit.Framework.Assert.IsTrue(clientContext.GetDisableLegacyBlockReaderLocal()); } HdfsDataInputStream stm = (HdfsDataInputStream)fs.Open(name); ByteBuffer actual = ByteBuffer.AllocateDirect(expected.Length - readOffset); IOUtils.SkipFully(stm, readOffset); actual.Limit(3); //Read a small number of bytes first. int nread = stm.Read(actual); actual.Limit(nread + 2); nread += stm.Read(actual); // Read across chunk boundary actual.Limit(Math.Min(actual.Capacity(), nread + 517)); nread += stm.Read(actual); CheckData(ArrayFromByteBuffer(actual), readOffset, expected, nread, "A few bytes" ); //Now read rest of it actual.Limit(actual.Capacity()); while (actual.HasRemaining()) { int nbytes = stm.Read(actual); if (nbytes < 0) { throw new EOFException("End of file reached before reading fully."); } nread += nbytes; } CheckData(ArrayFromByteBuffer(actual), readOffset, expected, "Read 3"); if (legacyShortCircuitFails) { NUnit.Framework.Assert.IsTrue(clientContext.GetDisableLegacyBlockReaderLocal()); } stm.Close(); }
/// <exception cref="System.IO.IOException"/> private bool CheckUnsupportedMethod(FileSystem fs, Path file, byte[] expected, int readOffset) { HdfsDataInputStream stm = (HdfsDataInputStream)fs.Open(file); ByteBuffer actual = ByteBuffer.AllocateDirect(expected.Length - readOffset); IOUtils.SkipFully(stm, readOffset); try { stm.Read(actual); } catch (NotSupportedException) { return(true); } return(false); }
public virtual void TestZeroCopyReadsNoFallback() { HdfsConfiguration conf = InitZeroCopyTest(); MiniDFSCluster cluster = null; Path TestPath = new Path("/a"); FSDataInputStream fsIn = null; int TestFileLength = 3 * BlockSize; FileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, 7567L); try { DFSTestUtil.WaitReplication(fs, TestPath, (short)1); } catch (Exception e) { NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: " + e); } catch (TimeoutException e) { NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: " + e); } fsIn = fs.Open(TestPath); byte[] original = new byte[TestFileLength]; IOUtils.ReadFully(fsIn, original, 0, TestFileLength); fsIn.Close(); fsIn = fs.Open(TestPath); HdfsDataInputStream dfsIn = (HdfsDataInputStream)fsIn; ByteBuffer result; try { result = dfsIn.Read(null, BlockSize + 1, EnumSet.NoneOf <ReadOption>()); NUnit.Framework.Assert.Fail("expected UnsupportedOperationException"); } catch (NotSupportedException) { } // expected result = dfsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(BlockSize, result.Remaining()); NUnit.Framework.Assert.AreEqual(BlockSize, dfsIn.GetReadStatistics().GetTotalBytesRead ()); NUnit.Framework.Assert.AreEqual(BlockSize, dfsIn.GetReadStatistics().GetTotalZeroCopyBytesRead ()); Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray (result)); } finally { if (fsIn != null) { fsIn.Close(); } if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
//check the file /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal static void CheckFile(Path p, int expectedsize, Configuration conf) { //open the file with another user account string username = UserGroupInformation.GetCurrentUser().GetShortUserName() + "_" + ++userCount; UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new string[] { "supergroup" }); FileSystem fs = DFSTestUtil.GetFileSystemAs(ugi, conf); HdfsDataInputStream @in = (HdfsDataInputStream)fs.Open(p); //Check visible length NUnit.Framework.Assert.IsTrue(@in.GetVisibleLength() >= expectedsize); //Able to read? for (int i = 0; i < expectedsize; i++) { NUnit.Framework.Assert.AreEqual(unchecked ((byte)i), unchecked ((byte)@in.Read())); } @in.Close(); }