public virtual void TestNegativeSeek()
{
instream = GetFileSystem().Open(smallSeekFile);
Assert.Equal(0, instream.GetPos());
try
{
instream.Seek(-1);
long p = instream.GetPos();
Log.Warn("Seek to -1 returned a position of " + p);
int result = instream.Read();
NUnit.Framework.Assert.Fail("expected an exception, got data " + result + " at a position of "
+ p);
}
catch (EOFException e)
{
//bad seek -expected
HandleExpectedException(e);
}
catch (IOException e)
{
//bad seek -expected, but not as preferred as an EOFException
HandleRelaxedException("a negative seek", "EOFException", e);
}
Assert.Equal(0, instream.GetPos());
}
public virtual void TestAppendLessThanChecksumChunk()
{
byte[] buf = new byte[1024];
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NumDataNodes
(1).Build();
cluster.WaitActive();
try
{
using (DistributedFileSystem fs = cluster.GetFileSystem())
{
int len1 = 200;
int len2 = 300;
Path p = new Path("/foo");
FSDataOutputStream @out = fs.Create(p);
@out.Write(buf, 0, len1);
@out.Close();
@out = fs.Append(p);
@out.Write(buf, 0, len2);
// flush but leave open
@out.Hflush();
// read data to verify the replica's content and checksum are correct
FSDataInputStream @in = fs.Open(p);
int length = @in.Read(0, buf, 0, len1 + len2);
NUnit.Framework.Assert.IsTrue(length > 0);
@in.Close();
@out.Close();
}
}
finally
{
cluster.Shutdown();
}
}
//
// validates that file encounters BlockMissingException
//
/// <exception cref="System.IO.IOException"/>
private void ValidateFile(FileSystem fileSys, Path name)
{
FSDataInputStream stm = fileSys.Open(name);
byte[] b = new byte[4192];
int num = 0;
bool gotException = false;
try
{
while (num >= 0)
{
num = stm.Read(b);
if (num < 0)
{
break;
}
}
}
catch (BlockMissingException)
{
gotException = true;
}
stm.Close();
NUnit.Framework.Assert.IsTrue("Expected BlockMissingException ", gotException);
}
/// <exception cref="System.Exception"/>
private void ReadTestFile(string testFileName)
{
Path filePath = new Path(testFileName);
FSDataInputStream istream = dfs.Open(filePath, 10240);
ByteBuffer buf = ByteBuffer.Allocate(10240);
int count = 0;
try
{
while (istream.Read(buf) > 0)
{
count += 1;
buf.Clear();
istream.Seek(istream.GetPos() + 5);
}
}
catch (IOException)
{
}
finally
{
// Ignore this it's probably a seek after eof.
istream.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void DoPread(FSDataInputStream stm, long position, byte[] buffer, int offset
, int length)
{
int nread = 0;
long totalRead = 0;
DFSInputStream dfstm = null;
if (stm.GetWrappedStream() is DFSInputStream)
{
dfstm = (DFSInputStream)(stm.GetWrappedStream());
totalRead = dfstm.GetReadStatistics().GetTotalBytesRead();
}
while (nread < length)
{
int nbytes = stm.Read(position + nread, buffer, offset + nread, length - nread);
NUnit.Framework.Assert.IsTrue("Error in pread", nbytes > 0);
nread += nbytes;
}
if (dfstm != null)
{
if (isHedgedRead)
{
NUnit.Framework.Assert.IsTrue("Expected read statistic to be incremented", length
<= dfstm.GetReadStatistics().GetTotalBytesRead() - totalRead);
}
else
{
NUnit.Framework.Assert.AreEqual("Expected read statistic to be incremented", length
, dfstm.GetReadStatistics().GetTotalBytesRead() - totalRead);
}
}
}
public virtual void TestRamDiskShortCircuitRead()
{
StartUpCluster(ReplFactor, new StorageType[] { StorageType.RamDisk, StorageType.Default }, 2 * BlockSize - 1, true);
// 1 replica + delta, SCR read
string MethodName = GenericTestUtils.GetMethodName();
int Seed = unchecked ((int)(0xFADED));
Path path = new Path("/" + MethodName + ".dat");
MakeRandomTestFile(path, BlockSize, true, Seed);
EnsureFileReplicasOnStorageType(path, StorageType.RamDisk);
// Sleep for a short time to allow the lazy writer thread to do its job
Sharpen.Thread.Sleep(3 * LazyWriterIntervalSec * 1000);
//assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
FSDataInputStream fis = fs.Open(path);
// Verify SCR read counters
try
{
fis = fs.Open(path);
byte[] buf = new byte[BufferLength];
fis.Read(0, buf, 0, BufferLength);
HdfsDataInputStream dfsis = (HdfsDataInputStream)fis;
NUnit.Framework.Assert.AreEqual(BufferLength, dfsis.GetReadStatistics().GetTotalBytesRead
());
NUnit.Framework.Assert.AreEqual(BufferLength, dfsis.GetReadStatistics().GetTotalShortCircuitBytesRead
());
}
finally
{
fis.Close();
fis = null;
}
}
/// <summary>
/// Test for the case where the client beings to read a long block, but doesn't
/// read bytes off the stream quickly.
/// </summary>
/// <remarks>
/// Test for the case where the client beings to read a long block, but doesn't
/// read bytes off the stream quickly. The datanode should time out sending the
/// chunks and the transceiver should die, even if it has a long keepalive.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestSlowReader()
{
// Set a client socket cache expiry time much longer than
// the datanode-side expiration time.
long ClientExpiryMs = 600000L;
Configuration clientConf = new Configuration(conf);
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testSlowReader");
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
// Restart the DN with a shorter write timeout.
MiniDFSCluster.DataNodeProperties props = cluster.StopDataNode(0);
props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketWriteTimeoutKey, WriteTimeout);
props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveKey, 120000);
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNode(props, true));
dn = cluster.GetDataNodes()[0];
// Wait for heartbeats to avoid a startup race where we
// try to write the block while the DN is still starting.
cluster.TriggerHeartbeats();
DFSTestUtil.CreateFile(fs, TestFile, 1024 * 1024 * 8L, (short)1, 0L);
FSDataInputStream stm = fs.Open(TestFile);
stm.Read();
AssertXceiverCount(1);
GenericTestUtils.WaitFor(new _Supplier_193(this), 500, 50000);
// DN should time out in sendChunks, and this should force
// the xceiver to exit.
IOUtils.CloseStream(stm);
}
/// <exception cref="System.IO.IOException"/>
public override int Read(byte[] b, int off, int len)
{
if ((off | len | (off + len) | (b.Length - (off + len))) < 0)
{
throw new IndexOutOfRangeException();
}
int n = (int)Math.Min(int.MaxValue, Math.Min(len, (end - pos)));
if (n == 0)
{
return(-1);
}
int ret = 0;
lock (@in)
{
@in.Seek(pos);
ret = @in.Read(b, off, n);
}
if (ret < 0)
{
end = pos;
return(-1);
}
pos += ret;
return(ret);
}
/// <exception cref="System.Exception"/>
public virtual void TestShmBasedStaleness()
{
BlockReaderTestUtil.EnableShortCircuitShmTracing();
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
Configuration conf = CreateShortCircuitConf("testShmBasedStaleness", sockDir);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
ShortCircuitCache cache = fs.GetClient().GetClientContext().GetShortCircuitCache(
);
string TestFile = "/test_file";
int TestFileLen = 8193;
int Seed = unchecked ((int)(0xFADED));
DFSTestUtil.CreateFile(fs, new Path(TestFile), TestFileLen, (short)1, Seed);
FSDataInputStream fis = fs.Open(new Path(TestFile));
int first = fis.Read();
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, new Path(TestFile));
NUnit.Framework.Assert.IsTrue(first != -1);
cache.Accept(new _CacheVisitor_502(block));
// Stop the Namenode. This will close the socket keeping the client's
// shared memory segment alive, and make it stale.
cluster.GetDataNodes()[0].Shutdown();
cache.Accept(new _CacheVisitor_518(block));
cluster.Shutdown();
sockDir.Close();
}
public virtual void TestReadClosedStream()
{
Path testFile = new Path("/testfile+2");
FSDataOutputStream os = hdfs.Create(testFile, true);
os.WriteBytes("0123456789");
os.Close();
// ByteRangeInputStream delays opens until reads. Make sure it doesn't
// open a closed stream that has never been opened
FSDataInputStream @in = hftpFs.Open(testFile);
@in.Close();
CheckClosedStream(@in);
CheckClosedStream(@in.GetWrappedStream());
// force the stream to connect and then close it
@in = hftpFs.Open(testFile);
int ch = @in.Read();
NUnit.Framework.Assert.AreEqual('0', ch);
@in.Close();
CheckClosedStream(@in);
CheckClosedStream(@in.GetWrappedStream());
// make sure seeking doesn't automagically reopen the stream
@in.Seek(4);
CheckClosedStream(@in);
CheckClosedStream(@in.GetWrappedStream());
}
/// <exception cref="System.IO.IOException"/>
public static void Check(FileSystem fs, Path p, long length)
{
int i = -1;
try
{
FileStatus status = fs.GetFileStatus(p);
FSDataInputStream @in = fs.Open(p);
if (@in.GetWrappedStream() is DFSInputStream)
{
long len = ((DFSInputStream)@in.GetWrappedStream()).GetFileLength();
NUnit.Framework.Assert.AreEqual(length, len);
}
else
{
NUnit.Framework.Assert.AreEqual(length, status.GetLen());
}
for (i++; i < length; i++)
{
NUnit.Framework.Assert.AreEqual(unchecked ((byte)i), unchecked ((byte)@in.Read()));
}
i = -(int)length;
NUnit.Framework.Assert.AreEqual(-1, @in.Read());
//EOF
@in.Close();
}
catch (IOException ioe)
{
throw new IOException("p=" + p + ", length=" + length + ", i=" + i, ioe);
}
}
/// <summary>
/// Check file content, reading as user
/// <paramref name="readingUser"/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal static void CheckFileContent(URI uri, Path name, byte[] expected, int readOffset
, string readingUser, Configuration conf, bool legacyShortCircuitFails)
{
// Ensure short circuit is enabled
DistributedFileSystem fs = GetFileSystem(readingUser, uri, conf);
ClientContext getClientContext = ClientContext.GetFromConf(conf);
if (legacyShortCircuitFails)
{
NUnit.Framework.Assert.IsFalse(getClientContext.GetDisableLegacyBlockReaderLocal(
));
}
FSDataInputStream stm = fs.Open(name);
byte[] actual = new byte[expected.Length - readOffset];
stm.ReadFully(readOffset, actual);
CheckData(actual, readOffset, expected, "Read 2");
stm.Close();
// Now read using a different API.
actual = new byte[expected.Length - readOffset];
stm = fs.Open(name);
IOUtils.SkipFully(stm, readOffset);
//Read a small number of bytes first.
int nread = stm.Read(actual, 0, 3);
nread += stm.Read(actual, nread, 2);
//Read across chunk boundary
nread += stm.Read(actual, nread, 517);
CheckData(actual, readOffset, expected, nread, "A few bytes");
//Now read rest of it
while (nread < actual.Length)
{
int nbytes = stm.Read(actual, nread, actual.Length - nread);
if (nbytes < 0)
{
throw new EOFException("End of file reached before reading fully.");
}
nread += nbytes;
}
CheckData(actual, readOffset, expected, "Read 3");
if (legacyShortCircuitFails)
{
NUnit.Framework.Assert.IsTrue(getClientContext.GetDisableLegacyBlockReaderLocal()
);
}
stm.Close();
}
public virtual void TestRedirect()
{
string dir = "/testRedirect/";
string filename = "file";
Path p = new Path(dir, filename);
string[] writeStrings = CreateStrings("write to webhdfs ", "write");
string[] appendStrings = CreateStrings("append to webhdfs ", "append");
//test create: create a file for each namenode
for (int i = 0; i < webhdfs.Length; i++)
{
FSDataOutputStream @out = webhdfs[i].Create(p);
@out.Write(Sharpen.Runtime.GetBytesForString(writeStrings[i]));
@out.Close();
}
for (int i_1 = 0; i_1 < webhdfs.Length; i_1++)
{
//check file length
long expected = writeStrings[i_1].Length;
NUnit.Framework.Assert.AreEqual(expected, webhdfs[i_1].GetFileStatus(p).GetLen());
}
//test read: check file content for each namenode
for (int i_2 = 0; i_2 < webhdfs.Length; i_2++)
{
FSDataInputStream @in = webhdfs[i_2].Open(p);
for (int c; (c = @in.Read()) != -1; j++)
{
NUnit.Framework.Assert.AreEqual(writeStrings[i_2][j], c);
}
@in.Close();
}
//test append: append to the file for each namenode
for (int i_3 = 0; i_3 < webhdfs.Length; i_3++)
{
FSDataOutputStream @out = webhdfs[i_3].Append(p);
@out.Write(Sharpen.Runtime.GetBytesForString(appendStrings[i_3]));
@out.Close();
}
for (int i_4 = 0; i_4 < webhdfs.Length; i_4++)
{
//check file length
long expected = writeStrings[i_4].Length + appendStrings[i_4].Length;
NUnit.Framework.Assert.AreEqual(expected, webhdfs[i_4].GetFileStatus(p).GetLen());
}
//test read: check file content for each namenode
for (int i_5 = 0; i_5 < webhdfs.Length; i_5++)
{
StringBuilder b = new StringBuilder();
FSDataInputStream @in = webhdfs[i_5].Open(p);
for (int c; (c = @in.Read()) != -1;)
{
b.Append((char)c);
}
int wlen = writeStrings[i_5].Length;
NUnit.Framework.Assert.AreEqual(writeStrings[i_5], b.Substring(0, wlen));
NUnit.Framework.Assert.AreEqual(appendStrings[i_5], b.Substring(wlen));
@in.Close();
}
}
public virtual void TestSeekBigFile()
{
Describe("Seek round a large file and verify the bytes are what is expected");
Path testSeekFile = Path("bigseekfile.txt");
byte[] block = ContractTestUtils.Dataset(65536, 0, 255);
ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block);
instream = GetFileSystem().Open(testSeekFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
instream.Seek(0);
int result = instream.Read();
Assert.Equal(0, result);
Assert.Equal(1, instream.Read());
Assert.Equal(2, instream.Read());
//do seek 32KB ahead
instream.Seek(32768);
Assert.Equal("@32768", block[32768], unchecked ((byte)instream.
Read()));
instream.Seek(40000);
Assert.Equal("@40000", block[40000], unchecked ((byte)instream.
Read()));
instream.Seek(8191);
Assert.Equal("@8191", block[8191], unchecked ((byte)instream.Read
()));
instream.Seek(0);
Assert.Equal("@0", 0, unchecked ((byte)instream.Read()));
}
public virtual void TestSeekZeroByteFile()
{
Describe("seek and read a 0 byte file");
instream = GetFileSystem().Open(zeroByteFile);
Assert.Equal(0, instream.GetPos());
//expect initial read to fai;
int result = instream.Read();
AssertMinusOne("initial byte read", result);
byte[] buffer = new byte[1];
//expect that seek to 0 works
instream.Seek(0);
//reread, expect same exception
result = instream.Read();
AssertMinusOne("post-seek byte read", result);
result = instream.Read(buffer, 0, 1);
AssertMinusOne("post-seek buffer read", result);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSeek()
{
Path dir = new Path("/test/testSeek");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir));
{
//test zero file size
Path zero = new Path(dir, "zero");
fs.Create(zero).Close();
int count = 0;
FSDataInputStream @in = fs.Open(zero);
for (; @in.Read() != -1; count++)
{
}
@in.Close();
NUnit.Framework.Assert.AreEqual(0, count);
}
byte[] mydata = new byte[1 << 20];
new Random().NextBytes(mydata);
Path p = new Path(dir, "file");
FSDataOutputStream @out = fs.Create(p, false, 4096, (short)3, 1L << 17);
@out.Write(mydata, 0, mydata.Length);
@out.Close();
int one_third = mydata.Length / 3;
int two_third = one_third * 2;
{
//test seek
int offset = one_third;
int len = mydata.Length - offset;
byte[] buf = new byte[len];
FSDataInputStream @in = fs.Open(p);
@in.Seek(offset);
//read all remaining data
@in.ReadFully(buf);
@in.Close();
for (int i = 0; i < buf.Length; i++)
{
NUnit.Framework.Assert.AreEqual("Position " + i + ", offset=" + offset + ", length="
+ len, mydata[i + offset], buf[i]);
}
}
{
//test position read (read the data after the two_third location)
int offset = two_third;
int len = mydata.Length - offset;
byte[] buf = new byte[len];
FSDataInputStream @in = fs.Open(p);
@in.ReadFully(offset, buf);
@in.Close();
for (int i = 0; i < buf.Length; i++)
{
NUnit.Framework.Assert.AreEqual("Position " + i + ", offset=" + offset + ", length="
+ len, mydata[i + offset], buf[i]);
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSkipWithVerifyChecksum()
{
int size = blockSize;
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "/tmp/testSkipWithVerifyChecksum._PORT"
);
DomainSocket.DisableBindPathValidation();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(
true).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// check that / exists
Path path = new Path("/");
NUnit.Framework.Assert.IsTrue("/ should be a directory", fs.GetFileStatus(path).IsDirectory
() == true);
byte[] fileData = AppendTestUtil.RandomBytes(seed, size * 3);
// create a new file in home directory. Do not close it.
Path file1 = new Path("filelocal.dat");
FSDataOutputStream stm = CreateFile(fs, file1, 1);
// write to file
stm.Write(fileData);
stm.Close();
// now test the skip function
FSDataInputStream instm = fs.Open(file1);
byte[] actual = new byte[fileData.Length];
// read something from the block first, otherwise BlockReaderLocal.skip()
// will not be invoked
int nread = instm.Read(actual, 0, 3);
long skipped = 2 * size + 3;
instm.Seek(skipped);
nread = instm.Read(actual, (int)(skipped + nread), 3);
instm.Close();
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestBlockReadZeroByteFile()
{
Describe("do a block read on a 0 byte file");
instream = GetFileSystem().Open(zeroByteFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
byte[] buffer = new byte[1];
int result = instream.Read(buffer, 0, 1);
AssertMinusOne("block read zero byte file", result);
}
public virtual void TestSeekFile()
{
Describe("basic seek operations");
instream = GetFileSystem().Open(smallSeekFile);
Assert.Equal(0, instream.GetPos());
//expect that seek to 0 works
instream.Seek(0);
int result = instream.Read();
Assert.Equal(0, result);
Assert.Equal(1, instream.Read());
Assert.Equal(2, instream.GetPos());
Assert.Equal(2, instream.Read());
Assert.Equal(3, instream.GetPos());
instream.Seek(128);
Assert.Equal(128, instream.GetPos());
Assert.Equal(128, instream.Read());
instream.Seek(63);
Assert.Equal(63, instream.Read());
}
public virtual void TestRamDiskEvictionWithShortCircuitReadHandle()
{
StartUpCluster(ReplFactor, new StorageType[] { StorageType.RamDisk, StorageType.Default }, (6 * BlockSize - 1), true);
// 5 replica + delta, SCR.
string MethodName = GenericTestUtils.GetMethodName();
Path path1 = new Path("/" + MethodName + ".01.dat");
Path path2 = new Path("/" + MethodName + ".02.dat");
int Seed = unchecked ((int)(0xFADED));
MakeRandomTestFile(path1, BlockSize, true, Seed);
EnsureFileReplicasOnStorageType(path1, StorageType.RamDisk);
// Sleep for a short time to allow the lazy writer thread to do its job.
// However the block replica should not be evicted from RAM_DISK yet.
Sharpen.Thread.Sleep(3 * LazyWriterIntervalSec * 1000);
// No eviction should happen as the free ratio is below the threshold
FSDataInputStream fis = fs.Open(path1);
try
{
// Keep and open read handle to path1 while creating path2
byte[] buf = new byte[BufferLength];
fis.Read(0, buf, 0, BufferLength);
// Create the 2nd file that will trigger RAM_DISK eviction.
MakeTestFile(path2, BlockSize * 2, true);
EnsureFileReplicasOnStorageType(path2, StorageType.RamDisk);
// Ensure path1 is still readable from the open SCR handle.
fis.Read(fis.GetPos(), buf, 0, BufferLength);
HdfsDataInputStream dfsis = (HdfsDataInputStream)fis;
NUnit.Framework.Assert.AreEqual(2 * BufferLength, dfsis.GetReadStatistics().GetTotalBytesRead
());
NUnit.Framework.Assert.AreEqual(2 * BufferLength, dfsis.GetReadStatistics().GetTotalShortCircuitBytesRead
());
}
finally
{
IOUtils.CloseQuietly(fis);
}
// After the open handle is closed, path1 should be evicted to DISK.
TriggerBlockReport();
EnsureFileReplicasOnStorageType(path1, StorageType.Default);
}
public virtual void TestOpenReadZeroByteFile()
{
Describe("create & read a 0 byte file");
Path path = Path("zero.txt");
ContractTestUtils.Touch(GetFileSystem(), path);
instream = GetFileSystem().Open(path);
Assert.Equal(0, instream.GetPos());
//expect initial read to fail
int result = instream.Read();
AssertMinusOne("initial byte read", result);
}
public virtual void TestSeek()
{
Path testFile = new Path("/testfile+1");
FSDataOutputStream @out = hdfs.Create(testFile, true);
@out.WriteBytes("0123456789");
@out.Close();
FSDataInputStream @in = hftpFs.Open(testFile);
@in.Seek(7);
NUnit.Framework.Assert.AreEqual('7', @in.Read());
@in.Close();
}
public virtual void TestSeekPastEndOfFileThenReseekAndRead()
{
Describe("do a seek past the EOF, then verify the stream recovers");
instream = GetFileSystem().Open(smallSeekFile);
//go just before the end. This may or may not fail; it may be delayed until the
//read
bool canSeekPastEOF = !GetContract().IsSupported(ContractOptions.RejectsSeekPastEof
, true);
try
{
instream.Seek(TestFileLen + 1);
//if this doesn't trigger, then read() is expected to fail
AssertMinusOne("read after seeking past EOF", instream.Read());
}
catch (EOFException e)
{
//This is an error iff the FS claims to be able to seek past the EOF
if (canSeekPastEOF)
{
//a failure wasn't expected
throw;
}
HandleExpectedException(e);
}
catch (IOException e)
{
//This is an error iff the FS claims to be able to seek past the EOF
if (canSeekPastEOF)
{
//a failure wasn't expected
throw;
}
HandleRelaxedException("a seek past the end of the file", "EOFException", e);
}
//now go back and try to read from a valid point in the file
instream.Seek(1);
Assert.True("Premature EOF", instream.Read() != -1);
}
/// <exception cref="System.IO.IOException"/>
protected internal override int ReadChunk(long pos, byte[] buf, int offset, int len
, byte[] checksum)
{
bool eof = false;
if (NeedChecksum())
{
System.Diagnostics.Debug.Assert(checksum != null);
// we have a checksum buffer
System.Diagnostics.Debug.Assert(checksum.Length % ChecksumSize == 0);
// it is sane length
System.Diagnostics.Debug.Assert(len >= bytesPerSum);
// we must read at least one chunk
int checksumsToRead = Math.Min(len / bytesPerSum, checksum.Length / ChecksumSize);
// number of checksums based on len to read
// size of checksum buffer
long checksumPos = GetChecksumFilePos(pos);
if (checksumPos != sums.GetPos())
{
sums.Seek(checksumPos);
}
int sumLenRead = sums.Read(checksum, 0, ChecksumSize * checksumsToRead);
if (sumLenRead >= 0 && sumLenRead % ChecksumSize != 0)
{
throw new ChecksumException("Checksum file not a length multiple of checksum size "
+ "in " + file + " at " + pos + " checksumpos: " + checksumPos + " sumLenread: "
+ sumLenRead, pos);
}
if (sumLenRead <= 0)
{
// we're at the end of the file
eof = true;
}
else
{
// Adjust amount of data to read based on how many checksum chunks we read
len = Math.Min(len, bytesPerSum * (sumLenRead / ChecksumSize));
}
}
if (pos != datas.GetPos())
{
datas.Seek(pos);
}
int nread = ReadFully(datas, buf, offset, len);
if (eof && nread > 0)
{
throw new ChecksumException("Checksum error: " + file + " at " + pos, pos);
}
return(nread);
}
/// <exception cref="System.IO.IOException"/>
private void TestEOF(MiniDFSCluster cluster, int fileLength)
{
FileSystem fs = cluster.GetFileSystem();
Path path = new Path("testEOF." + fileLength);
DFSTestUtil.CreateFile(fs, path, fileLength, (short)1, unchecked ((int)(0xBEEFBEEF
)));
FSDataInputStream fis = fs.Open(path);
ByteBuffer empty = ByteBuffer.Allocate(0);
// A read into an empty bytebuffer at the beginning of the file gives 0.
NUnit.Framework.Assert.AreEqual(0, fis.Read(empty));
fis.Seek(fileLength);
// A read into an empty bytebuffer at the end of the file gives -1.
NUnit.Framework.Assert.AreEqual(-1, fis.Read(empty));
if (fileLength > BlockSize)
{
fis.Seek(fileLength - BlockSize + 1);
ByteBuffer dbb = ByteBuffer.AllocateDirect(BlockSize);
NUnit.Framework.Assert.AreEqual(BlockSize - 1, fis.Read(dbb));
}
fis.Close();
}
// read a file using fetchBlockByteRange()
private bool CheckFile2(FSDataInputStream @in)
{
byte[] toRead = new byte[FileSize];
try
{
NUnit.Framework.Assert.AreEqual("Cannot read file", toRead.Length, @in.Read(0, toRead
, 0, toRead.Length));
}
catch (IOException)
{
return(false);
}
return(CheckFile(toRead));
}
public virtual void TestPositionedBulkReadDoesntChangePosition()
{
Describe("verify that a positioned read does not change the getPos() value");
Path testSeekFile = Path("bigseekfile.txt");
byte[] block = ContractTestUtils.Dataset(65536, 0, 255);
ContractTestUtils.CreateFile(GetFileSystem(), testSeekFile, false, block);
instream = GetFileSystem().Open(testSeekFile);
instream.Seek(39999);
Assert.True(-1 != instream.Read());
Assert.Equal(40000, instream.GetPos());
byte[] readBuffer = new byte[256];
instream.Read(128, readBuffer, 0, readBuffer.Length);
//have gone back
Assert.Equal(40000, instream.GetPos());
//content is the same too
Assert.Equal("@40000", block[40000], unchecked ((byte)instream.
Read()));
//now verify the picked up data
for (int i = 0; i < 256; i++)
{
Assert.Equal("@" + i, block[i + 128], readBuffer[i]);
}
}
/// <summary>Load from a Hadoop filesystem</summary>
/// <param name="fs">filesystem</param>
/// <param name="path">path</param>
/// <returns>a loaded CD</returns>
/// <exception cref="System.IO.IOException">IO problems</exception>
/// <exception cref="System.IO.EOFException">if not enough bytes were read in</exception>
/// <exception cref="Org.Codehaus.Jackson.JsonParseException">parse problems</exception>
/// <exception cref="Org.Codehaus.Jackson.Map.JsonMappingException">O/J mapping problems
/// </exception>
public virtual T Load(FileSystem fs, Path path)
{
FileStatus status = fs.GetFileStatus(path);
long len = status.GetLen();
byte[] b = new byte[(int)len];
FSDataInputStream dataInputStream = fs.Open(path);
int count = dataInputStream.Read(b);
if (count != len)
{
throw new EOFException(path.ToString() + ": read finished prematurely");
}
return(FromBytes(path.ToString(), b));
}
public virtual void TestPipelineRecoveryForLastBlock()
{
DFSClientFaultInjector faultInjector = Org.Mockito.Mockito.Mock <DFSClientFaultInjector
>();
DFSClientFaultInjector oldInjector = DFSClientFaultInjector.instance;
DFSClientFaultInjector.instance = faultInjector;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsClientBlockWriteLocatefollowingblockRetriesKey, 3);
MiniDFSCluster cluster = null;
try
{
int numDataNodes = 3;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fileSys = cluster.GetFileSystem();
Path file = new Path("dataprotocol1.dat");
Org.Mockito.Mockito.When(faultInjector.FailPacket()).ThenReturn(true);
DFSTestUtil.CreateFile(fileSys, file, 68000000L, (short)numDataNodes, 0L);
// At this point, NN should have accepted only valid replicas.
// Read should succeed.
FSDataInputStream @in = fileSys.Open(file);
try
{
int c = @in.Read();
}
catch (BlockMissingException)
{
// Test will fail with BlockMissingException if NN does not update the
// replica state based on the latest report.
NUnit.Framework.Assert.Fail("Block is missing because the file was closed with" +
" corrupt replicas.");
}
}
finally
{
DFSClientFaultInjector.instance = oldInjector;
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Test that when we have an uncache request, and the client refuses to release
/// the replica for a long time, we will un-mlock it.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestRevocation()
{
Assume.AssumeTrue(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows);
BlockReaderTestUtil.EnableHdfsCachingTracing();
BlockReaderTestUtil.EnableShortCircuitShmTracing();
Configuration conf = GetDefaultConf();
// Set a really short revocation timeout.
conf.SetLong(DFSConfigKeys.DfsDatanodeCacheRevocationTimeoutMs, 250L);
// Poll very often
conf.SetLong(DFSConfigKeys.DfsDatanodeCacheRevocationPollingMs, 2L);
MiniDFSCluster cluster = null;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
// Create and cache a file.
string TestFile = "/test_file2";
DFSTestUtil.CreateFile(dfs, new Path(TestFile), BlockSize, (short)1, unchecked ((int
)(0xcafe)));
dfs.AddCachePool(new CachePoolInfo("pool"));
long cacheDirectiveId = dfs.AddCacheDirective(new CacheDirectiveInfo.Builder().SetPool
("pool").SetPath(new Path(TestFile)).SetReplication((short)1).Build());
FsDatasetSpi <object> fsd = cluster.GetDataNodes()[0].GetFSDataset();
DFSTestUtil.VerifyExpectedCacheUsage(BlockSize, 1, fsd);
// Mmap the file.
FSDataInputStream @in = dfs.Open(new Path(TestFile));
ByteBuffer buf = @in.Read(null, BlockSize, EnumSet.NoneOf <ReadOption>());
// Attempt to uncache file. The file should get uncached.
Log.Info("removing cache directive {}", cacheDirectiveId);
dfs.RemoveCacheDirective(cacheDirectiveId);
Log.Info("finished removing cache directive {}", cacheDirectiveId);
Sharpen.Thread.Sleep(1000);
DFSTestUtil.VerifyExpectedCacheUsage(0, 0, fsd);
// Cleanup
@in.ReleaseBuffer(buf);
@in.Close();
cluster.Shutdown();
}
