/// <summary>Test manual failover failback for one namespace</summary>
/// <param name="cluster">single process test cluster</param>
/// <param name="conf">cluster configuration</param>
/// <param name="nsIndex">namespace index starting from zero</param>
/// <exception cref="System.Exception"/>
private void TestManualFailoverFailback(MiniDFSCluster cluster, Configuration conf
, int nsIndex)
{
int nn0 = 2 * nsIndex;
int nn1 = 2 * nsIndex + 1;
cluster.TransitionToActive(nn0);
Log.Info("Starting with NN 0 active in namespace " + nsIndex);
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
fs.Mkdirs(TestDir);
Log.Info("Failing over to NN 1 in namespace " + nsIndex);
cluster.TransitionToStandby(nn0);
cluster.TransitionToActive(nn1);
NUnit.Framework.Assert.IsTrue(fs.Exists(TestDir));
DFSTestUtil.WriteFile(fs, TestFilePath, TestFileData);
Log.Info("Failing over to NN 0 in namespace " + nsIndex);
cluster.TransitionToStandby(nn1);
cluster.TransitionToActive(nn0);
NUnit.Framework.Assert.IsTrue(fs.Exists(TestDir));
NUnit.Framework.Assert.AreEqual(TestFileData, DFSTestUtil.ReadFile(fs, TestFilePath
));
Log.Info("Removing test file");
fs.Delete(TestDir, true);
NUnit.Framework.Assert.IsFalse(fs.Exists(TestDir));
Log.Info("Failing over to NN 1 in namespace " + nsIndex);
cluster.TransitionToStandby(nn0);
cluster.TransitionToActive(nn1);
NUnit.Framework.Assert.IsFalse(fs.Exists(TestDir));
}
/// <summary>Test that the client respects its keepalive timeout.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestClientResponsesKeepAliveTimeout()
{
Configuration clientConf = new Configuration(conf);
// Set a client socket cache expiry time much shorter than
// the datanode-side expiration time.
long ClientExpiryMs = 10L;
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testClientResponsesKeepAliveTimeout"
);
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache();
DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L);
// Clients that write aren't currently re-used.
NUnit.Framework.Assert.AreEqual(0, peerCache.Size());
AssertXceiverCount(0);
// Reads the file, so we should get a
// cached socket, and should have an xceiver on the other side.
DFSTestUtil.ReadFile(fs, TestFile);
NUnit.Framework.Assert.AreEqual(1, peerCache.Size());
AssertXceiverCount(1);
// Sleep for a bit longer than the client keepalive timeout.
Sharpen.Thread.Sleep(ClientExpiryMs + 50);
// Taking out a peer which is expired should give a null.
Peer peer = peerCache.Get(dn.GetDatanodeId(), false);
NUnit.Framework.Assert.IsTrue(peer == null);
// The socket cache is now empty.
NUnit.Framework.Assert.AreEqual(0, peerCache.Size());
}
public virtual void TestEncryptedAppend()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
SetEncryptionConfigKeys(conf);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
FileSystem fs = GetFileSystem(conf);
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText + PlainText, DFSTestUtil.ReadFile(fs, TestPath
));
fs.Close();
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestLongLivedWriteClientAfterRestart()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
SetEncryptionConfigKeys(conf);
cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = GetFileSystem(conf);
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
// Restart the NN and DN, after which the client's encryption key will no
// longer be valid.
cluster.RestartNameNode();
NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes());
cluster.WaitActive();
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText + PlainText, DFSTestUtil.ReadFile(fs, TestPath
));
fs.Close();
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestBlockReportsWhileFileBeingWritten()
{
FSDataOutputStream @out = fs.Create(TestFilePath);
try
{
AppendTestUtil.Write(@out, 0, 10);
@out.Hflush();
// Block report will include the RBW replica, but will be
// queued on the StandbyNode.
cluster.TriggerBlockReports();
}
finally
{
IOUtils.CloseStream(@out);
}
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
// Verify that no replicas are marked corrupt, and that the
// file is readable from the failed-over standby.
BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager());
BlockManagerTestUtil.UpdateState(nn2.GetNamesystem().GetBlockManager());
NUnit.Framework.Assert.AreEqual(0, nn1.GetNamesystem().GetCorruptReplicaBlocks());
NUnit.Framework.Assert.AreEqual(0, nn2.GetNamesystem().GetCorruptReplicaBlocks());
DFSTestUtil.ReadFile(fs, TestFilePath);
}
public virtual void EnsureInvalidBlockTokensAreRejected()
{
cluster.TransitionToActive(0);
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
DFSTestUtil.WriteFile(fs, TestPath, TestData);
NUnit.Framework.Assert.AreEqual(TestData, DFSTestUtil.ReadFile(fs, TestPath));
DFSClient dfsClient = DFSClientAdapter.GetDFSClient((DistributedFileSystem)fs);
DFSClient spyDfsClient = Org.Mockito.Mockito.Spy(dfsClient);
Org.Mockito.Mockito.DoAnswer(new _Answer_121()).When(spyDfsClient).GetLocatedBlocks
(Org.Mockito.Mockito.AnyString(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito
.AnyLong());
// This will make the token invalid, since the password
// won't match anymore
DFSClientAdapter.SetDFSClient((DistributedFileSystem)fs, spyDfsClient);
try
{
NUnit.Framework.Assert.AreEqual(TestData, DFSTestUtil.ReadFile(fs, TestPath));
NUnit.Framework.Assert.Fail("Shouldn't have been able to read a file with invalid block tokens"
);
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Could not obtain block", ioe);
}
}
public virtual void TestDisableCache()
{
HdfsConfiguration confWithoutCache = new HdfsConfiguration();
// Configure a new instance with no peer caching, ensure that it doesn't
// cache anything
confWithoutCache.SetInt(DFSConfigKeys.DfsClientSocketCacheCapacityKey, 0);
BlockReaderTestUtil util = new BlockReaderTestUtil(1, confWithoutCache);
Path testFile = new Path("/testConnCache.dat");
util.WriteFile(testFile, FileSize / 1024);
FileSystem fsWithoutCache = FileSystem.NewInstance(util.GetConf());
try
{
DFSTestUtil.ReadFile(fsWithoutCache, testFile);
NUnit.Framework.Assert.AreEqual(0, ((DistributedFileSystem)fsWithoutCache).dfs.GetClientContext
().GetPeerCache().Size());
}
finally
{
fsWithoutCache.Close();
util.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeRollingUpgradeWithRollback()
{
try
{
StartCluster();
// Create files in DFS.
Path testFile1 = new Path("/" + GenericTestUtils.GetMethodName() + ".01.dat");
DFSTestUtil.CreateFile(fs, testFile1, FileSize, ReplFactor, Seed);
string fileContents1 = DFSTestUtil.ReadFile(fs, testFile1);
StartRollingUpgrade();
FilePath blockFile = GetBlockForFile(testFile1, true);
FilePath trashFile = GetTrashFileForBlock(blockFile, false);
DeleteAndEnsureInTrash(testFile1, blockFile, trashFile);
// Now perform a rollback to restore DFS to the pre-rollback state.
RollbackRollingUpgrade();
// Ensure that block was restored from trash
EnsureTrashRestored(blockFile, trashFile);
// Ensure that files exist and restored file contents are the same.
System.Diagnostics.Debug.Assert((fs.Exists(testFile1)));
string fileContents2 = DFSTestUtil.ReadFile(fs, testFile1);
Assert.AssertThat(fileContents1, IS.Is(fileContents2));
}
finally
{
ShutdownCluster();
}
}
public virtual void TestClientThatDoesNotSupportEncryption()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
// Set short retry timeouts so this test runs faster
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = GetFileSystem(conf);
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
fs.Close();
cluster.Shutdown();
SetEncryptionConfigKeys(conf);
cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(false).ManageNameDfsDirs
(false).Format(false).StartupOption(HdfsServerConstants.StartupOption.Regular).Build
();
fs = GetFileSystem(conf);
DFSClient client = DFSClientAdapter.GetDFSClient((DistributedFileSystem)fs);
DFSClient spyClient = Org.Mockito.Mockito.Spy(client);
Org.Mockito.Mockito.DoReturn(false).When(spyClient).ShouldEncryptData();
DFSClientAdapter.SetDFSClient((DistributedFileSystem)fs, spyClient);
GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory
.GetLog(typeof(DataNode)));
try
{
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
if (resolverClazz != null && !resolverClazz.EndsWith("TestTrustedChannelResolver"
))
{
NUnit.Framework.Assert.Fail("Should not have been able to read without encryption enabled."
);
}
}
catch (IOException ioe)
{
GenericTestUtils.AssertExceptionContains("Could not obtain block:", ioe);
}
finally
{
logs.StopCapturing();
}
fs.Close();
if (resolverClazz == null)
{
GenericTestUtils.AssertMatches(logs.GetOutput(), "Failed to read expected encryption handshake from client at"
);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestCanReadData()
{
Path file1 = new Path("/fileToRead.dat");
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
BackupNode backup = null;
try
{
// Start NameNode and BackupNode
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(true).Build();
fileSys = cluster.GetFileSystem();
long txid = cluster.GetNameNodeRpc().GetTransactionID();
backup = StartBackupNode(conf, HdfsServerConstants.StartupOption.Backup, 1);
WaitCheckpointDone(cluster, txid);
// Setup dual NameNode configuration for DataNodes
string rpcAddrKeyPreffix = DFSConfigKeys.DfsNamenodeRpcAddressKey + ".bnCluster";
string nnAddr = cluster.GetNameNode().GetNameNodeAddressHostPortString();
conf.Get(DFSConfigKeys.DfsNamenodeRpcAddressKey);
string bnAddr = backup.GetNameNodeAddressHostPortString();
conf.Set(DFSConfigKeys.DfsNameservices, "bnCluster");
conf.Set(DFSConfigKeys.DfsNameserviceId, "bnCluster");
conf.Set(DFSConfigKeys.DfsHaNamenodesKeyPrefix + ".bnCluster", "nnActive, nnBackup"
);
conf.Set(rpcAddrKeyPreffix + ".nnActive", nnAddr);
conf.Set(rpcAddrKeyPreffix + ".nnBackup", bnAddr);
cluster.StartDataNodes(conf, 3, true, HdfsServerConstants.StartupOption.Regular,
null);
DFSTestUtil.CreateFile(fileSys, file1, 8192, (short)3, 0);
// Read the same file from file systems pointing to NN and BN
FileSystem bnFS = FileSystem.Get(new Path("hdfs://" + bnAddr).ToUri(), conf);
string nnData = DFSTestUtil.ReadFile(fileSys, file1);
string bnData = DFSTestUtil.ReadFile(bnFS, file1);
NUnit.Framework.Assert.AreEqual("Data read from BackupNode and NameNode is not the same."
, nnData, bnData);
}
catch (IOException e)
{
Log.Error("Error in TestBackupNode: ", e);
NUnit.Framework.Assert.IsTrue(e.GetLocalizedMessage(), false);
}
finally
{
if (fileSys != null)
{
fileSys.Close();
}
if (backup != null)
{
backup.Stop();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestOperationsWhileInSafeMode()
{
Path file1 = new Path("/file1");
NUnit.Framework.Assert.IsFalse(dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeGet
));
DFSTestUtil.CreateFile(fs, file1, 1024, (short)1, 0);
NUnit.Framework.Assert.IsTrue("Could not enter SM", dfs.SetSafeMode(HdfsConstants.SafeModeAction
.SafemodeEnter));
RunFsFun("Set quota while in SM", new _FSRun_319(file1));
RunFsFun("Set perm while in SM", new _FSRun_325(file1));
RunFsFun("Set owner while in SM", new _FSRun_331(file1));
RunFsFun("Set repl while in SM", new _FSRun_337(file1));
RunFsFun("Append file while in SM", new _FSRun_343(file1));
RunFsFun("Truncate file while in SM", new _FSRun_349(file1));
RunFsFun("Delete file while in SM", new _FSRun_355(file1));
RunFsFun("Rename file while in SM", new _FSRun_361(file1));
RunFsFun("Set time while in SM", new _FSRun_367(file1));
RunFsFun("modifyAclEntries while in SM", new _FSRun_373(file1));
RunFsFun("removeAclEntries while in SM", new _FSRun_379(file1));
RunFsFun("removeDefaultAcl while in SM", new _FSRun_385(file1));
RunFsFun("removeAcl while in SM", new _FSRun_391(file1));
RunFsFun("setAcl while in SM", new _FSRun_397(file1));
RunFsFun("setXAttr while in SM", new _FSRun_403(file1));
RunFsFun("removeXAttr while in SM", new _FSRun_409(file1));
try
{
DFSTestUtil.ReadFile(fs, file1);
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("Set times failed while in SM");
}
try
{
fs.GetAclStatus(file1);
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("getAclStatus failed while in SM");
}
// Test access
UserGroupInformation ugiX = UserGroupInformation.CreateRemoteUser("userX");
FileSystem myfs = ugiX.DoAs(new _PrivilegedExceptionAction_429(this));
myfs.Access(file1, FsAction.Read);
try
{
myfs.Access(file1, FsAction.Write);
NUnit.Framework.Assert.Fail("The access call should have failed.");
}
catch (AccessControlException)
{
}
// expected
NUnit.Framework.Assert.IsFalse("Could not leave SM", dfs.SetSafeMode(HdfsConstants.SafeModeAction
.SafemodeLeave));
}
public virtual void TestRamDiskEvictionIsLru()
{
int NumPaths = 5;
StartUpCluster(true, NumPaths + EvictionLowWatermark);
string MethodName = GenericTestUtils.GetMethodName();
Path[] paths = new Path[NumPaths * 2];
for (int i = 0; i < paths.Length; i++)
{
paths[i] = new Path("/" + MethodName + "." + i + ".dat");
}
for (int i_1 = 0; i_1 < NumPaths; i_1++)
{
MakeTestFile(paths[i_1], BlockSize, true);
}
// Sleep for a short time to allow the lazy writer thread to do its job.
Sharpen.Thread.Sleep(3 * LazyWriterIntervalSec * 1000);
for (int i_2 = 0; i_2 < NumPaths; ++i_2)
{
EnsureFileReplicasOnStorageType(paths[i_2], StorageType.RamDisk);
}
// Open the files for read in a random order.
AList <int> indexes = new AList <int>(NumPaths);
for (int i_3 = 0; i_3 < NumPaths; ++i_3)
{
indexes.AddItem(i_3);
}
Collections.Shuffle(indexes);
for (int i_4 = 0; i_4 < NumPaths; ++i_4)
{
Log.Info("Touching file " + paths[indexes[i_4]]);
DFSTestUtil.ReadFile(fs, paths[indexes[i_4]]);
}
// Create an equal number of new files ensuring that the previous
// files are evicted in the same order they were read.
for (int i_5 = 0; i_5 < NumPaths; ++i_5)
{
MakeTestFile(paths[i_5 + NumPaths], BlockSize, true);
TriggerBlockReport();
Sharpen.Thread.Sleep(3000);
EnsureFileReplicasOnStorageType(paths[i_5 + NumPaths], StorageType.RamDisk);
EnsureFileReplicasOnStorageType(paths[indexes[i_5]], StorageType.Default);
for (int j = i_5 + 1; j < NumPaths; ++j)
{
EnsureFileReplicasOnStorageType(paths[indexes[j]], StorageType.RamDisk);
}
}
VerifyRamDiskJMXMetric("RamDiskBlocksWrite", NumPaths * 2);
VerifyRamDiskJMXMetric("RamDiskBlocksWriteFallback", 0);
VerifyRamDiskJMXMetric("RamDiskBytesWrite", BlockSize * NumPaths * 2);
VerifyRamDiskJMXMetric("RamDiskBlocksReadHits", NumPaths);
VerifyRamDiskJMXMetric("RamDiskBlocksEvicted", NumPaths);
VerifyRamDiskJMXMetric("RamDiskBlocksEvictedWithoutRead", 0);
VerifyRamDiskJMXMetric("RamDiskBlocksDeletedBeforeLazyPersisted", 0);
}
public virtual void TestEncryptedReadWithRC4()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = GetFileSystem(conf);
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
FileChecksum checksum = fs.GetFileChecksum(TestPath);
fs.Close();
cluster.Shutdown();
SetEncryptionConfigKeys(conf);
// It'll use 3DES by default, but we set it to rc4 here.
conf.Set(DFSConfigKeys.DfsDataEncryptionAlgorithmKey, "rc4");
cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(false).ManageNameDfsDirs
(false).Format(false).StartupOption(HdfsServerConstants.StartupOption.Regular).Build
();
fs = GetFileSystem(conf);
GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory
.GetLog(typeof(SaslDataTransferServer)));
GenericTestUtils.LogCapturer logs1 = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory
.GetLog(typeof(DataTransferSaslUtil)));
try
{
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
NUnit.Framework.Assert.AreEqual(checksum, fs.GetFileChecksum(TestPath));
}
finally
{
logs.StopCapturing();
logs1.StopCapturing();
}
fs.Close();
if (resolverClazz == null)
{
// Test client and server negotiate cipher option
GenericTestUtils.AssertDoesNotMatch(logs.GetOutput(), "Server using cipher suite"
);
// Check the IOStreamPair
GenericTestUtils.AssertDoesNotMatch(logs1.GetOutput(), "Creating IOStreamPair of CryptoInputStream and CryptoOutputStream."
);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestDnFencing()
{
// Create a file with replication level 3.
DFSTestUtil.CreateFile(fs, TestFilePath, 30 * SmallBlock, (short)3, 1L);
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestFilePath);
// Drop its replication count to 1, so it becomes over-replicated.
// Then compute the invalidation of the extra blocks and trigger
// heartbeats so the invalidations are flushed to the DNs.
nn1.GetRpcServer().SetReplication(TestFile, (short)1);
BlockManagerTestUtil.ComputeInvalidationWork(nn1.GetNamesystem().GetBlockManager(
));
cluster.TriggerHeartbeats();
// Transition nn2 to active even though nn1 still thinks it's active.
Banner("Failing to NN2 but let NN1 continue to think it's active");
NameNodeAdapter.AbortEditLogs(nn1);
NameNodeAdapter.EnterSafeMode(nn1, false);
cluster.TransitionToActive(1);
// Check that the standby picked up the replication change.
NUnit.Framework.Assert.AreEqual(1, nn2.GetRpcServer().GetFileInfo(TestFile).GetReplication
());
// Dump some info for debugging purposes.
Banner("NN2 Metadata immediately after failover");
DoMetasave(nn2);
Banner("Triggering heartbeats and block reports so that fencing is completed");
cluster.TriggerHeartbeats();
cluster.TriggerBlockReports();
Banner("Metadata after nodes have all block-reported");
DoMetasave(nn2);
// Force a rescan of postponedMisreplicatedBlocks.
BlockManager nn2BM = nn2.GetNamesystem().GetBlockManager();
BlockManagerTestUtil.CheckHeartbeat(nn2BM);
BlockManagerTestUtil.RescanPostponedMisreplicatedBlocks(nn2BM);
// The blocks should no longer be postponed.
NUnit.Framework.Assert.AreEqual(0, nn2.GetNamesystem().GetPostponedMisreplicatedBlocks
());
// Wait for NN2 to enact its deletions (replication monitor has to run, etc)
BlockManagerTestUtil.ComputeInvalidationWork(nn2.GetNamesystem().GetBlockManager(
));
cluster.TriggerHeartbeats();
HATestUtil.WaitForDNDeletions(cluster);
cluster.TriggerDeletionReports();
NUnit.Framework.Assert.AreEqual(0, nn2.GetNamesystem().GetUnderReplicatedBlocks()
);
NUnit.Framework.Assert.AreEqual(0, nn2.GetNamesystem().GetPendingReplicationBlocks
());
Banner("Making sure the file is still readable");
FileSystem fs2 = cluster.GetFileSystem(1);
DFSTestUtil.ReadFile(fs2, TestFilePath);
Banner("Waiting for the actual block files to get deleted from DNs.");
WaitForTrueReplication(cluster, block, 1);
}
// return the initial state of the configuration
/// <summary>
/// Test for the case where one of the DNs in the pipeline is in the
/// process of doing a block report exactly when the block is closed.
/// </summary>
/// <remarks>
/// Test for the case where one of the DNs in the pipeline is in the
/// process of doing a block report exactly when the block is closed.
/// In this case, the block report becomes delayed until after the
/// block is marked completed on the NN, and hence it reports an RBW
/// replica for a COMPLETE block. Such a report should not be marked
/// corrupt.
/// This is a regression test for HDFS-2791.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestOneReplicaRbwReportArrivesAfterBlockCompleted()
{
CountDownLatch brFinished = new CountDownLatch(1);
GenericTestUtils.DelayAnswer delayer = new _DelayAnswer_579(brFinished, Log);
// inform the test that our block report went through.
string MethodName = GenericTestUtils.GetMethodName();
Path filePath = new Path("/" + MethodName + ".dat");
// Start a second DN for this test -- we're checking
// what happens when one of the DNs is slowed for some reason.
ReplFactor = 2;
StartDNandWait(null, false);
NameNode nn = cluster.GetNameNode();
FSDataOutputStream @out = fs.Create(filePath, ReplFactor);
try
{
AppendTestUtil.Write(@out, 0, 10);
@out.Hflush();
// Set up a spy so that we can delay the block report coming
// from this node.
DataNode dn = cluster.GetDataNodes()[0];
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.SpyOnBposToNN(dn,
nn);
Org.Mockito.Mockito.DoAnswer(delayer).When(spy).BlockReport(Org.Mockito.Mockito.AnyObject
<DatanodeRegistration>(), Org.Mockito.Mockito.AnyString(), Org.Mockito.Mockito.AnyObject
<StorageBlockReport[]>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
// Force a block report to be generated. The block report will have
// an RBW replica in it. Wait for the RPC to be sent, but block
// it before it gets to the NN.
dn.ScheduleAllBlockReport(0);
delayer.WaitForCall();
}
finally
{
IOUtils.CloseStream(@out);
}
// Now that the stream is closed, the NN will have the block in COMPLETE
// state.
delayer.Proceed();
brFinished.Await();
// Verify that no replicas are marked corrupt, and that the
// file is still readable.
BlockManagerTestUtil.UpdateState(nn.GetNamesystem().GetBlockManager());
NUnit.Framework.Assert.AreEqual(0, nn.GetNamesystem().GetCorruptReplicaBlocks());
DFSTestUtil.ReadFile(fs, filePath);
// Ensure that the file is readable even from the DN that we futzed with.
cluster.StopDataNode(1);
DFSTestUtil.ReadFile(fs, filePath);
}
public virtual void TestWriteNewFile()
{
OutputStream fos = new AtomicFileOutputStream(DstFile);
NUnit.Framework.Assert.IsFalse(DstFile.Exists());
fos.Write(Sharpen.Runtime.GetBytesForString(TestString));
fos.Flush();
NUnit.Framework.Assert.IsFalse(DstFile.Exists());
fos.Close();
NUnit.Framework.Assert.IsTrue(DstFile.Exists());
string readBackData = DFSTestUtil.ReadFile(DstFile);
NUnit.Framework.Assert.AreEqual(TestString, readBackData);
}
public virtual void TestEarlierVersionEditLog()
{
Configuration conf = new HdfsConfiguration();
string tarFile = Runtime.GetProperty("test.cache.data", "build/test/cache") + "/"
+ Hadoop10MultiblockTgz;
string testDir = PathUtils.GetTestDirName(GetType());
FilePath dfsDir = new FilePath(testDir, "image-1.0");
if (dfsDir.Exists() && !FileUtil.FullyDelete(dfsDir))
{
throw new IOException("Could not delete dfs directory '" + dfsDir + "'");
}
FileUtil.UnTar(new FilePath(tarFile), new FilePath(testDir));
FilePath nameDir = new FilePath(dfsDir, "name");
GenericTestUtils.AssertExists(nameDir);
FilePath dataDir = new FilePath(dfsDir, "data");
GenericTestUtils.AssertExists(dataDir);
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameDir.GetAbsolutePath());
conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dataDir.GetAbsolutePath());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Format(
false).ManageDataDfsDirs(false).ManageNameDfsDirs(false).NumDataNodes(1).StartupOption
(HdfsServerConstants.StartupOption.Upgrade).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
Path testPath = new Path("/user/todd/4blocks");
// Read it without caring about the actual data within - we just need
// to make sure that the block states and locations are OK.
DFSTestUtil.ReadFile(fs, testPath);
// Ensure that we can append to it - if the blocks were in some funny
// state we'd get some kind of issue here.
FSDataOutputStream stm = fs.Append(testPath);
try
{
stm.Write(1);
}
finally
{
IOUtils.CloseStream(stm);
}
}
finally
{
cluster.Shutdown();
}
}
/// <summary>Tests DataTransferProtocol with the given client configuration.</summary>
/// <param name="conf">client configuration</param>
/// <exception cref="System.IO.IOException">if there is an I/O error</exception>
private void DoTest(HdfsConfiguration conf)
{
fs = FileSystem.Get(cluster.GetURI(), conf);
FileSystemTestHelper.CreateFile(fs, Path, NumBlocks, BlockSize);
Assert.AssertArrayEquals(FileSystemTestHelper.GetFileData(NumBlocks, BlockSize),
Sharpen.Runtime.GetBytesForString(DFSTestUtil.ReadFile(fs, Path), "UTF-8"));
BlockLocation[] blockLocations = fs.GetFileBlockLocations(Path, 0, long.MaxValue);
NUnit.Framework.Assert.IsNotNull(blockLocations);
NUnit.Framework.Assert.AreEqual(NumBlocks, blockLocations.Length);
foreach (BlockLocation blockLocation in blockLocations)
{
NUnit.Framework.Assert.IsNotNull(blockLocation.GetHosts());
NUnit.Framework.Assert.AreEqual(3, blockLocation.GetHosts().Length);
}
}
public virtual void TestOverwriteFile()
{
NUnit.Framework.Assert.IsTrue("Creating empty dst file", DstFile.CreateNewFile());
OutputStream fos = new AtomicFileOutputStream(DstFile);
NUnit.Framework.Assert.IsTrue("Empty file still exists", DstFile.Exists());
fos.Write(Sharpen.Runtime.GetBytesForString(TestString));
fos.Flush();
// Original contents still in place
NUnit.Framework.Assert.AreEqual(string.Empty, DFSTestUtil.ReadFile(DstFile));
fos.Close();
// New contents replace original file
string readBackData = DFSTestUtil.ReadFile(DstFile);
NUnit.Framework.Assert.AreEqual(TestString, readBackData);
}
public virtual void TestCorruptBlockRereplicatedAcrossRacks()
{
Configuration conf = GetConf();
short ReplicationFactor = 2;
int fileLen = 512;
Path filePath = new Path("/testFile");
// Datanodes are spread across two racks
string[] racks = new string[] { "/rack1", "/rack1", "/rack2", "/rack2" };
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(racks.Length
).Racks(racks).Build();
FSNamesystem ns = cluster.GetNameNode().GetNamesystem();
try
{
// Create a file with one block with a replication factor of 2
FileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, filePath, fileLen, ReplicationFactor, 1L);
string fileContent = DFSTestUtil.ReadFile(fs, filePath);
ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, filePath);
DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0);
// Corrupt a replica of the block
int dnToCorrupt = DFSTestUtil.FirstDnWithBlock(cluster, b);
NUnit.Framework.Assert.IsTrue(cluster.CorruptReplica(dnToCorrupt, b));
// Restart the datanode so blocks are re-scanned, and the corrupt
// block is detected.
cluster.RestartDataNode(dnToCorrupt);
// Wait for the namenode to notice the corrupt replica
DFSTestUtil.WaitCorruptReplicas(fs, ns, filePath, b, 1);
// The rack policy is still respected
DFSTestUtil.WaitForReplication(cluster, b, 2, ReplicationFactor, 0);
// Ensure all replicas are valid (the corrupt replica may not
// have been cleaned up yet).
for (int i = 0; i < racks.Length; i++)
{
string blockContent = cluster.ReadBlockOnDataNode(i, b);
if (blockContent != null && i != dnToCorrupt)
{
NUnit.Framework.Assert.AreEqual("Corrupt replica", fileContent, blockContent);
}
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestFencingStress()
{
HAStressTestHarness harness = new HAStressTestHarness();
harness.conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000);
harness.conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1);
harness.conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1);
MiniDFSCluster cluster = harness.StartCluster();
try
{
cluster.WaitActive();
cluster.TransitionToActive(0);
FileSystem fs = harness.GetFailoverFs();
MultithreadedTestUtil.TestContext togglers = new MultithreadedTestUtil.TestContext
();
for (int i = 0; i < NumThreads; i++)
{
Path p = new Path("/test-" + i);
DFSTestUtil.CreateFile(fs, p, BlockSize * 10, (short)3, (long)i);
togglers.AddThread(new TestDNFencingWithReplication.ReplicationToggler(togglers,
fs, p));
}
// Start a separate thread which will make sure that replication
// happens quickly by triggering deletion reports and replication
// work calculation frequently.
harness.AddReplicationTriggerThread(500);
harness.AddFailoverThread(5000);
harness.StartThreads();
togglers.StartThreads();
togglers.WaitFor(Runtime);
togglers.Stop();
harness.StopThreads();
// CHeck that the files can be read without throwing
for (int i_1 = 0; i_1 < NumThreads; i_1++)
{
Path p = new Path("/test-" + i_1);
DFSTestUtil.ReadFile(fs, p);
}
}
finally
{
System.Console.Error.WriteLine("===========================\n\n\n\n");
harness.Shutdown();
}
}
public virtual void TestLongLivedClient()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = GetFileSystem(conf);
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
FileChecksum checksum = fs.GetFileChecksum(TestPath);
fs.Close();
cluster.Shutdown();
SetEncryptionConfigKeys(conf);
cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(false).ManageNameDfsDirs
(false).Format(false).StartupOption(HdfsServerConstants.StartupOption.Regular).Build
();
BlockTokenSecretManager btsm = cluster.GetNamesystem().GetBlockManager().GetBlockTokenSecretManager
();
btsm.SetKeyUpdateIntervalForTesting(2 * 1000);
btsm.SetTokenLifetime(2 * 1000);
btsm.ClearAllKeysForTesting();
fs = GetFileSystem(conf);
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
NUnit.Framework.Assert.AreEqual(checksum, fs.GetFileChecksum(TestPath));
// Sleep for 15 seconds, after which the encryption key will no longer be
// valid. It needs to be a few multiples of the block token lifetime,
// since several block tokens are valid at any given time (the current
// and the last two, by default.)
Log.Info("Sleeping so that encryption keys expire...");
Sharpen.Thread.Sleep(15 * 1000);
Log.Info("Done sleeping.");
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
NUnit.Framework.Assert.AreEqual(checksum, fs.GetFileChecksum(TestPath));
fs.Close();
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private void TestEncryptedWrite(int numDns)
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
SetEncryptionConfigKeys(conf);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDns).Build();
FileSystem fs = GetFileSystem(conf);
GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory
.GetLog(typeof(SaslDataTransferServer)));
GenericTestUtils.LogCapturer logs1 = GenericTestUtils.LogCapturer.CaptureLogs(LogFactory
.GetLog(typeof(DataTransferSaslUtil)));
try
{
WriteTestDataToFile(fs);
}
finally
{
logs.StopCapturing();
logs1.StopCapturing();
}
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
fs.Close();
if (resolverClazz == null)
{
// Test client and server negotiate cipher option
GenericTestUtils.AssertDoesNotMatch(logs.GetOutput(), "Server using cipher suite"
);
// Check the IOStreamPair
GenericTestUtils.AssertDoesNotMatch(logs1.GetOutput(), "Creating IOStreamPair of CryptoInputStream and CryptoOutputStream."
);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestSendDataPacketMetrics()
{
Configuration conf = new HdfsConfiguration();
int interval = 1;
conf.Set(DFSConfigKeys.DfsMetricsPercentilesIntervalsKey, string.Empty + interval
);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
// Create and read a 1 byte file
Path tmpfile = new Path("/tmp.txt");
DFSTestUtil.CreateFile(fs, tmpfile, (long)1, (short)1, 1L);
DFSTestUtil.ReadFile(fs, tmpfile);
IList <DataNode> datanodes = cluster.GetDataNodes();
NUnit.Framework.Assert.AreEqual(datanodes.Count, 1);
DataNode datanode = datanodes[0];
MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(datanode.GetMetrics().Name());
// Expect 2 packets, 1 for the 1 byte read, 1 for the empty packet
// signaling the end of the block
MetricsAsserts.AssertCounter("SendDataPacketTransferNanosNumOps", (long)2, rb);
MetricsAsserts.AssertCounter("SendDataPacketBlockedOnNetworkNanosNumOps", (long)2
, rb);
// Wait for at least 1 rollover
Sharpen.Thread.Sleep((interval + 1) * 1000);
// Check that the sendPacket percentiles rolled to non-zero values
string sec = interval + "s";
MetricsAsserts.AssertQuantileGauges("SendDataPacketBlockedOnNetworkNanos" + sec,
rb);
MetricsAsserts.AssertQuantileGauges("SendDataPacketTransferNanos" + sec, rb);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestDataNodeTimeSpend()
{
Configuration conf = new HdfsConfiguration();
SimulatedFSDataset.SetFactory(conf);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
IList <DataNode> datanodes = cluster.GetDataNodes();
NUnit.Framework.Assert.AreEqual(datanodes.Count, 1);
DataNode datanode = datanodes[0];
MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(datanode.GetMetrics().Name());
long LongFileLen = 1024 * 1024 * 10;
long startWriteValue = MetricsAsserts.GetLongCounter("TotalWriteTime", rb);
long startReadValue = MetricsAsserts.GetLongCounter("TotalReadTime", rb);
for (int x = 0; x < 50; x++)
{
DFSTestUtil.CreateFile(fs, new Path("/time.txt." + x), LongFileLen, (short)1, Time
.MonotonicNow());
}
for (int x_1 = 0; x_1 < 50; x_1++)
{
string s = DFSTestUtil.ReadFile(fs, new Path("/time.txt." + x_1));
}
MetricsRecordBuilder rbNew = MetricsAsserts.GetMetrics(datanode.GetMetrics().Name
());
long endWriteValue = MetricsAsserts.GetLongCounter("TotalWriteTime", rbNew);
long endReadValue = MetricsAsserts.GetLongCounter("TotalReadTime", rbNew);
NUnit.Framework.Assert.IsTrue(endReadValue > startReadValue);
NUnit.Framework.Assert.IsTrue(endWriteValue > startWriteValue);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestWithCheckpoint()
{
Path path = new Path("/test");
DoWriteAndAbort(fs, path);
fs.Delete(new Path("/test/test"), true);
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
cluster.RestartNameNode(true);
// read snapshot file after restart
string test2snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.GetSnapshotPath(path.ToString(), "s1/test/test2");
DFSTestUtil.ReadFile(fs, new Path(test2snapshotPath));
string test3snapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.GetSnapshotPath(path.ToString(), "s1/test/test3");
DFSTestUtil.ReadFile(fs, new Path(test3snapshotPath));
}
/// <summary>Regression test for HDFS-3357.</summary>
/// <remarks>
/// Regression test for HDFS-3357. Check that the datanode is respecting
/// its configured keepalive timeout.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeRespectsKeepAliveTimeout()
{
Configuration clientConf = new Configuration(conf);
// Set a client socket cache expiry time much longer than
// the datanode-side expiration time.
long ClientExpiryMs = 60000L;
clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs
);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testDatanodeRespectsKeepAliveTimeout"
);
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache();
DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L);
// Clients that write aren't currently re-used.
NUnit.Framework.Assert.AreEqual(0, peerCache.Size());
AssertXceiverCount(0);
// Reads the file, so we should get a
// cached socket, and should have an xceiver on the other side.
DFSTestUtil.ReadFile(fs, TestFile);
NUnit.Framework.Assert.AreEqual(1, peerCache.Size());
AssertXceiverCount(1);
// Sleep for a bit longer than the keepalive timeout
// and make sure the xceiver died.
Sharpen.Thread.Sleep(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveDefault + 50);
AssertXceiverCount(0);
// The socket is still in the cache, because we don't
// notice that it's closed until we try to read
// from it again.
NUnit.Framework.Assert.AreEqual(1, peerCache.Size());
// Take it out of the cache - reading should
// give an EOF.
Peer peer = peerCache.Get(dn.GetDatanodeId(), false);
NUnit.Framework.Assert.IsNotNull(peer);
NUnit.Framework.Assert.AreEqual(-1, peer.GetInputStream().Read());
}
/// <exception cref="System.Exception"/>
public virtual void TestManyClosedSocketsInCache()
{
// Make a small file
Configuration clientConf = new Configuration(conf);
clientConf.Set(DFSConfigKeys.DfsClientContext, "testManyClosedSocketsInCache");
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI()
, clientConf);
PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache();
DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L);
// Insert a bunch of dead sockets in the cache, by opening
// many streams concurrently, reading all of the data,
// and then closing them.
InputStream[] stms = new InputStream[5];
try
{
for (int i = 0; i < stms.Length; i++)
{
stms[i] = fs.Open(TestFile);
}
foreach (InputStream stm in stms)
{
IOUtils.CopyBytes(stm, new IOUtils.NullOutputStream(), 1024);
}
}
finally
{
IOUtils.Cleanup(null, stms);
}
NUnit.Framework.Assert.AreEqual(5, peerCache.Size());
// Let all the xceivers timeout
Sharpen.Thread.Sleep(1500);
AssertXceiverCount(0);
// Client side still has the sockets cached
NUnit.Framework.Assert.AreEqual(5, peerCache.Size());
// Reading should not throw an exception.
DFSTestUtil.ReadFile(fs, TestFile);
}
/*
* Try to read the files inside snapshot but renamed to different file and
* deleted after restarting post checkpoint. refer HDFS-5427
*/
/// <exception cref="System.Exception"/>
public virtual void TestReadRenamedSnapshotFileWithCheckpoint()
{
Path foo = new Path("/foo");
Path foo2 = new Path("/foo2");
hdfs.Mkdirs(foo);
hdfs.Mkdirs(foo2);
hdfs.AllowSnapshot(foo);
hdfs.AllowSnapshot(foo2);
Path bar = new Path(foo, "bar");
Path bar2 = new Path(foo2, "bar");
DFSTestUtil.CreateFile(hdfs, bar, 100, (short)2, 100024L);
hdfs.CreateSnapshot(foo, "s1");
// rename to another snapshottable directory and take snapshot
NUnit.Framework.Assert.IsTrue(hdfs.Rename(bar, bar2));
hdfs.CreateSnapshot(foo2, "s2");
// delete the original renamed file to make sure blocks are not updated by
// the original file
NUnit.Framework.Assert.IsTrue(hdfs.Delete(bar2, true));
// checkpoint
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
// restart namenode to load snapshot files from fsimage
cluster.RestartNameNode(true);
// file in first snapshot
string barSnapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.GetSnapshotPath(foo.ToString(), "s1/bar");
DFSTestUtil.ReadFile(hdfs, new Path(barSnapshotPath));
// file in second snapshot after rename+delete
string bar2SnapshotPath = Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot
.GetSnapshotPath(foo2.ToString(), "s2/bar");
DFSTestUtil.ReadFile(hdfs, new Path(bar2SnapshotPath));
}
public virtual void TestEncryptedAppendRequiringBlockTransfer()
{
MiniDFSCluster cluster = null;
try
{
Configuration conf = new Configuration();
SetEncryptionConfigKeys(conf);
// start up 4 DNs
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
FileSystem fs = GetFileSystem(conf);
// Create a file with replication 3, so its block is on 3 / 4 DNs.
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText, DFSTestUtil.ReadFile(fs, TestPath));
// Shut down one of the DNs holding a block replica.
FSDataInputStream @in = fs.Open(TestPath);
IList <LocatedBlock> locatedBlocks = DFSTestUtil.GetAllBlocks(@in);
@in.Close();
NUnit.Framework.Assert.AreEqual(1, locatedBlocks.Count);
NUnit.Framework.Assert.AreEqual(3, locatedBlocks[0].GetLocations().Length);
DataNode dn = cluster.GetDataNode(locatedBlocks[0].GetLocations()[0].GetIpcPort()
);
dn.Shutdown();
// Reopen the file for append, which will need to add another DN to the
// pipeline and in doing so trigger a block transfer.
WriteTestDataToFile(fs);
NUnit.Framework.Assert.AreEqual(PlainText + PlainText, DFSTestUtil.ReadFile(fs, TestPath
));
fs.Close();
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}