public virtual void TestShortCircuitReadMetaFileCorruption()
{
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
StartUpCluster(true, 1 + EvictionLowWatermark, true, false);
DoShortCircuitReadMetaFileCorruptionTest();
}
public virtual void Before()
{
Assume.AssumeThat(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows, CoreMatchers.EqualTo
(true));
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
}
private static Configuration CreateShortCircuitConf(string testName, TemporarySocketDirectory
sockDir)
{
Configuration conf = new Configuration();
conf.Set(DFSConfigKeys.DfsClientContext, testName);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 4096);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), testName
).GetAbsolutePath());
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, false);
DFSInputStream.tcpReadsDisabledForTesting = true;
DomainSocket.DisableBindPathValidation();
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
return(conf);
}
public virtual void Before()
{
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
}
public virtual void Init()
{
DomainSocket.DisableBindPathValidation();
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
}
/// <exception cref="System.IO.IOException"/>
public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest
test, bool checksum, long readahead)
{
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
MiniDFSCluster cluster = null;
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum
);
conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest
.BytesPerChecksum);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead);
test.SetConfiguration(conf);
FileInputStream dataIn = null;
FileInputStream metaIn = null;
Path TestPath = new Path("/a");
long RandomSeed = 4567L;
BlockReaderLocal blockReaderLocal = null;
FSDataInputStream fsIn = null;
byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength];
FileSystem fs = null;
ShortCircuitShm shm = null;
RandomAccessFile raf = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
);
fsIn.Close();
fsIn = null;
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath);
FilePath dataFile = cluster.GetBlockFile(0, block);
FilePath metaFile = cluster.GetBlockMetadataFile(0, block);
ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache
();
cluster.Shutdown();
cluster = null;
test.Setup(dataFile, checksum);
FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile)
, new FileInputStream(metaFile) };
dataIn = streams[0];
metaIn = streams[1];
ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId
());
raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID.
RandomUUID().ToString()), "rw");
raf.SetLength(8192);
FileInputStream shmStream = new FileInputStream(raf.GetFD());
shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream);
ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache
, Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block))
);
blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename
(TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy
(new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build();
dataIn = null;
metaIn = null;
test.DoTest(blockReaderLocal, original);
// BlockReaderLocal should not alter the file position.
NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position());
NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position());
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
if (dataIn != null)
{
dataIn.Close();
}
if (metaIn != null)
{
metaIn.Close();
}
if (blockReaderLocal != null)
{
blockReaderLocal.Close();
}
if (shm != null)
{
shm.Free();
}
if (raf != null)
{
raf.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestReadLink()
{
// Create TEST_DIR\dir1\file1.txt
//
FilePath dir1 = new FilePath(TestDir, "dir1");
Assert.True(dir1.Mkdirs());
FilePath file1 = new FilePath(dir1, "file1.txt");
Assert.True(file1.CreateNewFile());
FilePath dirLink = new FilePath(TestDir, "dlink");
FilePath fileLink = new FilePath(TestDir, "flink");
// Next create a directory symlink to dir1 and a file
// symlink to file1.txt.
//
Shell.ExecCommand(Shell.Winutils, "symlink", dirLink.ToString(), dir1.ToString());
Shell.ExecCommand(Shell.Winutils, "symlink", fileLink.ToString(), file1.ToString(
));
// Read back the two links and ensure we get what we expected.
//
string readLinkOutput = Shell.ExecCommand(Shell.Winutils, "readlink", dirLink.ToString
());
Assert.AssertThat(readLinkOutput, CoreMatchers.EqualTo(dir1.ToString()));
readLinkOutput = Shell.ExecCommand(Shell.Winutils, "readlink", fileLink.ToString(
));
Assert.AssertThat(readLinkOutput, CoreMatchers.EqualTo(file1.ToString()));
// Try a few invalid inputs and verify we get an ExitCodeException for each.
//
try
{
// No link name specified.
//
Shell.ExecCommand(Shell.Winutils, "readlink", string.Empty);
NUnit.Framework.Assert.Fail("Failed to get Shell.ExitCodeException when reading bad symlink"
);
}
catch (Shell.ExitCodeException ece)
{
Assert.AssertThat(ece.GetExitCode(), CoreMatchers.Is(1));
}
try
{
// Bad link name.
//
Shell.ExecCommand(Shell.Winutils, "readlink", "ThereIsNoSuchLink");
NUnit.Framework.Assert.Fail("Failed to get Shell.ExitCodeException when reading bad symlink"
);
}
catch (Shell.ExitCodeException ece)
{
Assert.AssertThat(ece.GetExitCode(), CoreMatchers.Is(1));
}
try
{
// Non-symlink directory target.
//
Shell.ExecCommand(Shell.Winutils, "readlink", dir1.ToString());
NUnit.Framework.Assert.Fail("Failed to get Shell.ExitCodeException when reading bad symlink"
);
}
catch (Shell.ExitCodeException ece)
{
Assert.AssertThat(ece.GetExitCode(), CoreMatchers.Is(1));
}
try
{
// Non-symlink file target.
//
Shell.ExecCommand(Shell.Winutils, "readlink", file1.ToString());
NUnit.Framework.Assert.Fail("Failed to get Shell.ExitCodeException when reading bad symlink"
);
}
catch (Shell.ExitCodeException ece)
{
Assert.AssertThat(ece.GetExitCode(), CoreMatchers.Is(1));
}
try
{
// Too many parameters.
//
Shell.ExecCommand(Shell.Winutils, "readlink", "a", "b");
NUnit.Framework.Assert.Fail("Failed to get Shell.ExitCodeException with bad parameters"
);
}
catch (Shell.ExitCodeException ece)
{
Assert.AssertThat(ece.GetExitCode(), CoreMatchers.Is(1));
}
}
