public virtual void TestUnbufferClosesSockets()
{
Configuration conf = new Configuration();
// Set a new ClientContext. This way, we will have our own PeerCache,
// rather than sharing one with other unit tests.
conf.Set(DFSConfigKeys.DfsClientContext, "testUnbufferClosesSocketsContext");
// Disable short-circuit reads. With short-circuit, we wouldn't hold open a
// TCP socket.
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false);
// Set a really long socket timeout to avoid test timing issues.
conf.SetLong(DFSConfigKeys.DfsClientSocketTimeoutKey, 100000000L);
conf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, 100000000L);
MiniDFSCluster cluster = null;
FSDataInputStream stream = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.NewInstance(conf);
Path TestPath = new Path("/test1");
DFSTestUtil.CreateFile(dfs, TestPath, 128, (short)1, 1);
stream = dfs.Open(TestPath);
// Read a byte. This will trigger the creation of a block reader.
stream.Seek(2);
int b = stream.Read();
NUnit.Framework.Assert.IsTrue(-1 != b);
// The Peer cache should start off empty.
PeerCache cache = dfs.GetClient().GetClientContext().GetPeerCache();
NUnit.Framework.Assert.AreEqual(0, cache.Size());
// Unbuffer should clear the block reader and return the socket to the
// cache.
stream.Unbuffer();
stream.Seek(2);
NUnit.Framework.Assert.AreEqual(1, cache.Size());
int b2 = stream.Read();
NUnit.Framework.Assert.AreEqual(b, b2);
}
finally
{
if (stream != null)
{
IOUtils.Cleanup(null, stream);
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override int ReadChunk(long pos, byte[] buf, int offset, int len
, byte[] checksum)
{
bool eof = false;
if (NeedChecksum())
{
System.Diagnostics.Debug.Assert(checksum != null);
// we have a checksum buffer
System.Diagnostics.Debug.Assert(checksum.Length % ChecksumSize == 0);
// it is sane length
System.Diagnostics.Debug.Assert(len >= bytesPerSum);
// we must read at least one chunk
int checksumsToRead = Math.Min(len / bytesPerSum, checksum.Length / ChecksumSize);
// number of checksums based on len to read
// size of checksum buffer
long checksumPos = GetChecksumFilePos(pos);
if (checksumPos != sums.GetPos())
{
sums.Seek(checksumPos);
}
int sumLenRead = sums.Read(checksum, 0, ChecksumSize * checksumsToRead);
if (sumLenRead >= 0 && sumLenRead % ChecksumSize != 0)
{
throw new EOFException("Checksum file not a length multiple of checksum size " +
"in " + file + " at " + pos + " checksumpos: " + checksumPos + " sumLenread: " +
sumLenRead);
}
if (sumLenRead <= 0)
{
// we're at the end of the file
eof = true;
}
else
{
// Adjust amount of data to read based on how many checksum chunks we read
len = Math.Min(len, bytesPerSum * (sumLenRead / ChecksumSize));
}
}
if (pos != datas.GetPos())
{
datas.Seek(pos);
}
int nread = ReadFully(datas, buf, offset, len);
if (eof && nread > 0)
{
throw new ChecksumException("Checksum error: " + file + " at " + pos, pos);
}
return(nread);
}
/// <exception cref="System.IO.IOException"/>
private void VerifyRead(FSDataInputStream stm, byte[] fileContents, int seekOff,
int toRead)
{
byte[] @out = new byte[toRead];
stm.Seek(seekOff);
stm.ReadFully(@out);
byte[] expected = Arrays.CopyOfRange(fileContents, seekOff, seekOff + toRead);
if (!Arrays.Equals(@out, expected))
{
string s = "\nExpected: " + StringUtils.ByteToHexString(expected) + "\ngot: "
+ StringUtils.ByteToHexString(@out) + "\noff=" + seekOff + " len=" + toRead;
NUnit.Framework.Assert.Fail(s);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Map(Text key, LongWritable value, OutputCollector <K, LongWritable
> collector, Reporter reporter)
{
string name = key.ToString();
long size = value.Get();
long seed = long.Parse(name);
if (size == 0)
{
return;
}
reporter.SetStatus("opening " + name);
FSDataInputStream @in = fs.Open(new Path(DataDir, name));
try
{
for (int i = 0; i < SeeksPerFile; i++)
{
// generate a random position
long position = Math.Abs(random.NextLong()) % size;
// seek file to that position
reporter.SetStatus("seeking " + name);
@in.Seek(position);
byte b = @in.ReadByte();
// check that byte matches
byte checkByte = 0;
// advance random state to that position
random.SetSeed(seed);
for (int p = 0; p <= position; p += check.Length)
{
reporter.SetStatus("generating data for " + name);
if (fastCheck)
{
checkByte = unchecked ((byte)random.Next(byte.MaxValue));
}
else
{
random.NextBytes(check);
checkByte = check[(int)(position % check.Length)];
}
}
NUnit.Framework.Assert.AreEqual(b, checkByte);
}
}
finally
{
@in.Close();
}
}
/// <exception cref="System.IO.IOException"/>
internal override object DoIO(Reporter reporter, string name, long offset)
{
// open file
FSDataInputStream @in = null;
Path p = new Path(name);
try
{
@in = fs.Open(p);
}
catch (IOException)
{
return(name + "@(missing)");
}
@in.Seek(offset);
long actualSize = 0;
try
{
long blockSize = fs.GetDefaultBlockSize(p);
reporter.SetStatus("reading " + name + "@" + offset + "/" + blockSize);
for (int curSize = bufferSize; curSize == bufferSize && actualSize < blockSize; actualSize
+= curSize)
{
curSize = @in.Read(buffer, 0, bufferSize);
}
}
catch (IOException)
{
Log.Info("Corrupted block detected in \"" + name + "\" at " + offset);
return(name + "@" + offset);
}
finally
{
@in.Close();
}
return(actualSize);
}
/// <summary>
/// When mark() is used on BufferedInputStream, the request
/// size on the checksum file system can be small.
/// </summary>
/// <remarks>
/// When mark() is used on BufferedInputStream, the request
/// size on the checksum file system can be small. However,
/// checksum file system currently depends on the request size
/// >= bytesPerSum to work properly.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestTruncatedInputBug()
{
int ioBufSize = 512;
int fileSize = ioBufSize * 4;
int filePos = 0;
Configuration conf = new Configuration();
conf.SetInt("io.file.buffer.size", ioBufSize);
FileSystem fileSys = FileSystem.GetLocal(conf);
try
{
// First create a test input file.
Path testFile = new Path(TestRootDir, "HADOOP-1489");
WriteFile(fileSys, testFile, fileSize);
Assert.True(fileSys.Exists(testFile));
Assert.True(fileSys.GetFileStatus(testFile).GetLen() == fileSize
);
// Now read the file for ioBufSize bytes
FSDataInputStream @in = fileSys.Open(testFile, ioBufSize);
// seek beyond data buffered by open
filePos += ioBufSize * 2 + (ioBufSize - 10);
@in.Seek(filePos);
// read 4 more bytes before marking
for (int i = 0; i < 4; ++i)
{
if (@in.Read() == -1)
{
break;
}
++filePos;
}
// Now set mark() to trigger the bug
// NOTE: in the fixed code, mark() does nothing (not supported) and
// hence won't trigger this bug.
@in.Mark(1);
System.Console.Out.WriteLine("MARKED");
// Try to read the rest
while (filePos < fileSize)
{
if (@in.Read() == -1)
{
break;
}
++filePos;
}
@in.Close();
System.Console.Out.WriteLine("Read " + filePos + " bytes." + " file size=" + fileSize
);
Assert.True(filePos == fileSize);
}
finally
{
try
{
fileSys.Close();
}
catch (Exception)
{
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Seek(long p)
{
stream.Seek(p);
}
public virtual void Test2GBMmapLimit()
{
Assume.AssumeTrue(BlockReaderTestUtil.ShouldTestLargeFiles());
HdfsConfiguration conf = InitZeroCopyTest();
long TestFileLength = 2469605888L;
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "NULL");
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, TestFileLength);
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
string Context = "test2GBMmapLimit";
conf.Set(DFSConfigKeys.DfsClientContext, Context);
FSDataInputStream fsIn = null;
FSDataInputStream fsIn2 = null;
ByteBuffer buf1 = null;
ByteBuffer buf2 = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, unchecked ((int)(0xB
)));
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
fsIn = fs.Open(TestPath);
buf1 = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(1, buf1.Remaining());
fsIn.ReleaseBuffer(buf1);
buf1 = null;
fsIn.Seek(2147483640L);
buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(7, buf1.Remaining());
NUnit.Framework.Assert.AreEqual(int.MaxValue, buf1.Limit());
fsIn.ReleaseBuffer(buf1);
buf1 = null;
NUnit.Framework.Assert.AreEqual(2147483647L, fsIn.GetPos());
try
{
buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.Fail("expected UnsupportedOperationException");
}
catch (NotSupportedException)
{
}
// expected; can't read past 2GB boundary.
fsIn.Close();
fsIn = null;
// Now create another file with normal-sized blocks, and verify we
// can read past 2GB
Path TestPath2 = new Path("/b");
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 268435456L);
DFSTestUtil.CreateFile(fs, TestPath2, 1024 * 1024, TestFileLength, 268435456L, (short
)1, unchecked ((int)(0xA)));
fsIn2 = fs.Open(TestPath2);
fsIn2.Seek(2147483640L);
buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(8, buf2.Remaining());
NUnit.Framework.Assert.AreEqual(2147483648L, fsIn2.GetPos());
fsIn2.ReleaseBuffer(buf2);
buf2 = null;
buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(1024, buf2.Remaining());
NUnit.Framework.Assert.AreEqual(2147484672L, fsIn2.GetPos());
fsIn2.ReleaseBuffer(buf2);
buf2 = null;
}
finally
{
if (buf1 != null)
{
fsIn.ReleaseBuffer(buf1);
}
if (buf2 != null)
{
fsIn2.ReleaseBuffer(buf2);
}
IOUtils.Cleanup(null, fsIn, fsIn2);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestZeroCopyMmapCache()
{
HdfsConfiguration conf = InitZeroCopyTest();
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
int TestFileLength = 5 * BlockSize;
int RandomSeed = 23453;
string Context = "testZeroCopyMmapCacheContext";
FSDataInputStream fsIn = null;
ByteBuffer[] results = new ByteBuffer[] { null, null, null, null };
DistributedFileSystem fs = null;
conf.Set(DFSConfigKeys.DfsClientContext, Context);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
byte[] original = new byte[TestFileLength];
IOUtils.ReadFully(fsIn, original, 0, TestFileLength);
fsIn.Close();
fsIn = fs.Open(TestPath);
ShortCircuitCache cache = ClientContext.Get(Context, new DFSClient.Conf(conf)).GetShortCircuitCache
();
cache.Accept(new TestEnhancedByteBufferAccess.CountingVisitor(0, 5, 5, 0));
results[0] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums));
fsIn.Seek(0);
results[1] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums));
// The mmap should be of the first block of the file.
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fs, TestPath);
cache.Accept(new _CacheVisitor_373(firstBlock));
// The replica should not yet be evictable, since we have it open.
// Read more blocks.
results[2] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums));
results[3] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums));
// we should have 3 mmaps, 1 evictable
cache.Accept(new TestEnhancedByteBufferAccess.CountingVisitor(3, 5, 2, 0));
// After we close the cursors, the mmaps should be evictable for
// a brief period of time. Then, they should be closed (we're
// using a very quick timeout)
foreach (ByteBuffer buffer in results)
{
if (buffer != null)
{
fsIn.ReleaseBuffer(buffer);
}
}
fsIn.Close();
GenericTestUtils.WaitFor(new _Supplier_407(cache), 10, 60000);
cache.Accept(new TestEnhancedByteBufferAccess.CountingVisitor(0, -1, -1, -1));
fs.Close();
cluster.Shutdown();
}