/// <summary>Ask dfs client to read the file</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
private void DfsClientReadFile(Path corruptedFile)
{
DFSInputStream @in = dfs.dfs.Open(corruptedFile.ToUri().GetPath());
byte[] buf = new byte[buffersize];
int nRead = 0;
// total number of bytes read
try
{
do
{
nRead = @in.Read(buf, 0, buf.Length);
}while (nRead > 0);
}
catch (ChecksumException)
{
// caught ChecksumException if all replicas are bad, ignore and continue.
Log.Debug("DfsClientReadFile caught ChecksumException.");
}
catch (BlockMissingException)
{
// caught BlockMissingException, ignore.
Log.Debug("DfsClientReadFile caught BlockMissingException.");
}
}
/// <exception cref="System.IO.IOException"/>
private void DoPread(FSDataInputStream stm, long position, byte[] buffer, int offset
, int length)
{
int nread = 0;
long totalRead = 0;
DFSInputStream dfstm = null;
if (stm.GetWrappedStream() is DFSInputStream)
{
dfstm = (DFSInputStream)(stm.GetWrappedStream());
totalRead = dfstm.GetReadStatistics().GetTotalBytesRead();
}
while (nread < length)
{
int nbytes = stm.Read(position + nread, buffer, offset + nread, length - nread);
NUnit.Framework.Assert.IsTrue("Error in pread", nbytes > 0);
nread += nbytes;
}
if (dfstm != null)
{
if (isHedgedRead)
{
NUnit.Framework.Assert.IsTrue("Expected read statistic to be incremented", length
<= dfstm.GetReadStatistics().GetTotalBytesRead() - totalRead);
}
else
{
NUnit.Framework.Assert.AreEqual("Expected read statistic to be incremented", length
, dfstm.GetReadStatistics().GetTotalBytesRead() - totalRead);
}
}
}
/// <summary>(Optionally) seek to position, read and verify data.</summary>
/// <remarks>
/// (Optionally) seek to position, read and verify data.
/// Seek to specified position if pos is non-negative.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private void Pread(DFSInputStream @in, long pos, byte[] buffer, int offset, int length
, byte[] authenticData)
{
NUnit.Framework.Assert.IsTrue("Test buffer too small", buffer.Length >= offset +
length);
if (pos >= 0)
{
@in.Seek(pos);
}
Log.Info("Reading from file of size " + @in.GetFileLength() + " at offset " + @in
.GetPos());
while (length > 0)
{
int cnt = @in.Read(buffer, offset, length);
NUnit.Framework.Assert.IsTrue("Error in read", cnt > 0);
offset += cnt;
length -= cnt;
}
// Verify
for (int i = 0; i < length; ++i)
{
byte actual = buffer[i];
byte expect = authenticData[(int)pos + i];
NUnit.Framework.Assert.AreEqual("Read data mismatch at file offset " + (pos + i)
+ ". Expects " + expect + "; got " + actual, actual, expect);
}
}
public virtual void TestHedgedReadLoopTooManyTimes()
{
Configuration conf = new Configuration();
int numHedgedReadPoolThreads = 5;
int hedgedReadTimeoutMillis = 50;
conf.SetInt(DFSConfigKeys.DfsDfsclientHedgedReadThreadpoolSize, numHedgedReadPoolThreads
);
conf.SetLong(DFSConfigKeys.DfsDfsclientHedgedReadThresholdMillis, hedgedReadTimeoutMillis
);
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 0);
// Set up the InjectionHandler
DFSClientFaultInjector.instance = Org.Mockito.Mockito.Mock <DFSClientFaultInjector
>();
DFSClientFaultInjector injector = DFSClientFaultInjector.instance;
int sleepMs = 100;
Org.Mockito.Mockito.DoAnswer(new _Answer_296(hedgedReadTimeoutMillis, sleepMs)).When
(injector).FetchFromDatanodeException();
Org.Mockito.Mockito.DoAnswer(new _Answer_309(sleepMs)).When(injector).ReadFromDatanodeDelay
();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format(
true).Build();
DistributedFileSystem fileSys = cluster.GetFileSystem();
DFSClient dfsClient = fileSys.GetClient();
FSDataOutputStream output = null;
DFSInputStream input = null;
string filename = "/hedgedReadMaxOut.dat";
try
{
Path file = new Path(filename);
output = fileSys.Create(file, (short)2);
byte[] data = new byte[64 * 1024];
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Close();
byte[] buffer = new byte[64 * 1024];
input = dfsClient.Open(filename);
input.Read(0, buffer, 0, 1024);
input.Close();
NUnit.Framework.Assert.AreEqual(3, input.GetHedgedReadOpsLoopNumForTesting());
}
catch (BlockMissingException)
{
NUnit.Framework.Assert.IsTrue(false);
}
finally
{
Org.Mockito.Mockito.Reset(injector);
IOUtils.Cleanup(null, input);
IOUtils.Cleanup(null, output);
fileSys.Close();
cluster.Shutdown();
}
}
public virtual void TestDoGetShouldCloseTheDFSInputStreamIfResponseGetOutPutStreamThrowsAnyException
()
{
MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(1).Build();
try
{
Path testFile = CreateFile();
SetUpForDoGetTest(cluster, testFile);
Org.Mockito.Mockito.DoThrow(new IOException()).When(mockHttpServletResponse).GetOutputStream
();
DFSInputStream fsMock = Org.Mockito.Mockito.Mock <DFSInputStream>();
Org.Mockito.Mockito.DoReturn(fsMock).When(clientMock).Open(testFile.ToString());
Org.Mockito.Mockito.DoReturn(Sharpen.Extensions.ValueOf(4)).When(fsMock).GetFileLength
();
try
{
sfile.DoGet(mockHttpServletRequest, mockHttpServletResponse);
NUnit.Framework.Assert.Fail("Not throwing the IOException");
}
catch (IOException)
{
Org.Mockito.Mockito.Verify(clientMock, Org.Mockito.Mockito.AtLeastOnce()).Close();
}
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public override FSDataInputStream Load(DFSClientCache.DFSInputStreamCaheKey key)
{
DFSClient client = this._enclosing.GetDfsClient(key.userId);
DFSInputStream dis = client.Open(key.inodePath);
return(client.CreateWrappedInputStream(dis));
}
/// <summary>Positional read.</summary>
/// <exception cref="System.Exception"/>
private void PRead(int start, int len)
{
NUnit.Framework.Assert.IsTrue("Bad args: " + start + " + " + len + " should be <= "
+ fileSize, start + len <= fileSize);
DFSInputStream dis = testInfo.dis;
byte[] buf = new byte[len];
helper.PRead(dis, buf, start, len);
VerifyData("Pread data corrupted", buf, start, start + len);
}
/// <exception cref="System.IO.IOException"/>
public virtual int Read(DFSInputStream dis, byte[] target, int startOff, int len)
{
double p = rand.NextDouble();
if (p > CopyingProbability)
{
return bb.Read(dis, target, startOff, len);
}
else
{
return copy.Read(dis, target, startOff, len);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual int PRead(DFSInputStream dis, byte[] target, int startOff, int len
)
{
int cnt = 0;
while (cnt < len)
{
int read = dis.Read(startOff, target, cnt, len - cnt);
if (read == -1)
{
return read;
}
cnt += read;
}
return cnt;
}
/// <exception cref="System.IO.IOException"/>
public virtual int Read(DFSInputStream dis, byte[] target, int startOff, int len)
{
int cnt = 0;
lock (dis)
{
dis.Seek(startOff);
while (cnt < len)
{
int read = dis.Read(target, cnt, len - cnt);
if (read == -1)
{
return read;
}
cnt += read;
}
}
return cnt;
}
/// <exception cref="System.IO.IOException"/>
public virtual int Read(DFSInputStream dis, byte[] target, int startOff, int len)
{
ByteBuffer bb = ByteBuffer.AllocateDirect(target.Length);
int cnt = 0;
lock (dis)
{
dis.Seek(startOff);
while (cnt < len)
{
int read = dis.Read(bb);
if (read == -1)
{
return read;
}
cnt += read;
}
}
bb.Clear();
bb.Get(target);
return cnt;
}
/// <summary>DFS client read bytes starting from the specified position.</summary>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
/// <exception cref="System.IO.IOException"/>
private void DfsClientReadFileFromPosition(Path corruptedFile)
{
DFSInputStream @in = dfs.dfs.Open(corruptedFile.ToUri().GetPath());
byte[] buf = new byte[buffersize];
int startPosition = 2;
int nRead = 0;
// total number of bytes read
try
{
do
{
nRead = @in.Read(startPosition, buf, 0, buf.Length);
startPosition += buf.Length;
}while (nRead > 0);
}
catch (BlockMissingException)
{
Log.Debug("DfsClientReadFile caught BlockMissingException.");
}
}
public virtual void TestReadFromOneDN()
{
HdfsConfiguration configuration = new HdfsConfiguration();
// One of the goals of this test is to verify that we don't open more
// than one socket. So use a different client context, so that we
// get our own socket cache, rather than sharing with the other test
// instances. Also use a really long socket timeout so that nothing
// gets closed before we get around to checking the cache size at the end.
string contextName = "testReadFromOneDNContext";
configuration.Set(DFSConfigKeys.DfsClientContext, contextName);
configuration.SetLong(DFSConfigKeys.DfsClientSocketTimeoutKey, 100000000L);
BlockReaderTestUtil util = new BlockReaderTestUtil(1, configuration);
Path testFile = new Path("/testConnCache.dat");
byte[] authenticData = util.WriteFile(testFile, FileSize / 1024);
DFSClient client = new DFSClient(new IPEndPoint("localhost", util.GetCluster().GetNameNodePort
()), util.GetConf());
ClientContext cacheContext = ClientContext.Get(contextName, client.GetConf());
DFSInputStream @in = client.Open(testFile.ToString());
Log.Info("opened " + testFile.ToString());
byte[] dataBuf = new byte[BlockSize];
// Initial read
Pread(@in, 0, dataBuf, 0, dataBuf.Length, authenticData);
// Read again and verify that the socket is the same
Pread(@in, FileSize - dataBuf.Length, dataBuf, 0, dataBuf.Length, authenticData);
Pread(@in, 1024, dataBuf, 0, dataBuf.Length, authenticData);
// No seek; just read
Pread(@in, -1, dataBuf, 0, dataBuf.Length, authenticData);
Pread(@in, 64, dataBuf, 0, dataBuf.Length / 2, authenticData);
@in.Close();
client.Close();
NUnit.Framework.Assert.AreEqual(1, ClientContext.GetFromConf(configuration).GetPeerCache
().Size());
}
/// <exception cref="System.IO.IOException"/>
public HdfsDataInputStream(DFSInputStream @in)
: base(@in)
{
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
public override FSDataInputStream Open(Path f, int bufferSize)
{
DFSInputStream dfsis = dfs.Open(GetUriPath(f), bufferSize, verifyChecksum);
return(dfs.CreateWrappedInputStream(dfsis));
}
public virtual void TestLeaseAbort()
{
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
try
{
cluster.WaitActive();
NamenodeProtocols preSpyNN = cluster.GetNameNodeRpc();
NamenodeProtocols spyNN = Org.Mockito.Mockito.Spy(preSpyNN);
DFSClient dfs = new DFSClient(null, spyNN, conf, null);
byte[] buf = new byte[1024];
FSDataOutputStream c_out = CreateFsOut(dfs, dirString + "c");
c_out.Write(buf, 0, 1024);
c_out.Close();
DFSInputStream c_in = dfs.Open(dirString + "c");
FSDataOutputStream d_out = CreateFsOut(dfs, dirString + "d");
// stub the renew method.
Org.Mockito.Mockito.DoThrow(new RemoteException(typeof(SecretManager.InvalidToken
).FullName, "Your token is worthless")).When(spyNN).RenewLease(Matchers.AnyString
());
// We don't need to wait the lease renewer thread to act.
// call renewLease() manually.
// make it look like the soft limit has been exceeded.
LeaseRenewer originalRenewer = dfs.GetLeaseRenewer();
dfs.lastLeaseRenewal = Time.MonotonicNow() - HdfsConstants.LeaseSoftlimitPeriod -
1000;
try
{
dfs.RenewLease();
}
catch (IOException)
{
}
// Things should continue to work it passes hard limit without
// renewing.
try
{
d_out.Write(buf, 0, 1024);
Log.Info("Write worked beyond the soft limit as expected.");
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("Write failed.");
}
// make it look like the hard limit has been exceeded.
dfs.lastLeaseRenewal = Time.MonotonicNow() - HdfsConstants.LeaseHardlimitPeriod -
1000;
dfs.RenewLease();
// this should not work.
try
{
d_out.Write(buf, 0, 1024);
d_out.Close();
NUnit.Framework.Assert.Fail("Write did not fail even after the fatal lease renewal failure"
);
}
catch (IOException e)
{
Log.Info("Write failed as expected. ", e);
}
// If aborted, the renewer should be empty. (no reference to clients)
Sharpen.Thread.Sleep(1000);
NUnit.Framework.Assert.IsTrue(originalRenewer.IsEmpty());
// unstub
Org.Mockito.Mockito.DoNothing().When(spyNN).RenewLease(Matchers.AnyString());
// existing input streams should work
try
{
int num = c_in.Read(buf, 0, 1);
if (num != 1)
{
NUnit.Framework.Assert.Fail("Failed to read 1 byte");
}
c_in.Close();
}
catch (IOException e)
{
Log.Error("Read failed with ", e);
NUnit.Framework.Assert.Fail("Read after lease renewal failure failed");
}
// new file writes should work.
try
{
c_out = CreateFsOut(dfs, dirString + "c");
c_out.Write(buf, 0, 1024);
c_out.Close();
}
catch (IOException e)
{
Log.Error("Write failed with ", e);
NUnit.Framework.Assert.Fail("Write failed");
}
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual int PRead(DFSInputStream dis, byte[] target, int startOff, int len
)
{
// No pRead for bb read path
return Read(dis, target, startOff, len);
}