/// <summary>Duplicate the given block on all volumes.</summary>
/// <param name="blockId"/>
/// <exception cref="System.IO.IOException"/>
private void DuplicateBlock(long blockId)
{
lock (fds)
{
ReplicaInfo b = FsDatasetTestUtil.FetchReplicaInfo(fds, bpid, blockId);
foreach (FsVolumeSpi v in fds.GetVolumes())
{
if (v.GetStorageID().Equals(b.GetVolume().GetStorageID()))
{
continue;
}
// Volume without a copy of the block. Make a copy now.
FilePath sourceBlock = b.GetBlockFile();
FilePath sourceMeta = b.GetMetaFile();
string sourceRoot = b.GetVolume().GetBasePath();
string destRoot = v.GetBasePath();
string relativeBlockPath = new FilePath(sourceRoot).ToURI().Relativize(sourceBlock
.ToURI()).GetPath();
string relativeMetaPath = new FilePath(sourceRoot).ToURI().Relativize(sourceMeta.
ToURI()).GetPath();
FilePath destBlock = new FilePath(destRoot, relativeBlockPath);
FilePath destMeta = new FilePath(destRoot, relativeMetaPath);
destBlock.GetParentFile().Mkdirs();
FileUtils.CopyFile(sourceBlock, destBlock);
FileUtils.CopyFile(sourceMeta, destMeta);
if (destBlock.Exists() && destMeta.Exists())
{
Log.Info("Copied " + sourceBlock + " ==> " + destBlock);
Log.Info("Copied " + sourceMeta + " ==> " + destMeta);
}
}
}
}
/// <exception cref="System.Exception"/>
internal TestContext(Configuration conf, int numNameServices)
{
this.numNameServices = numNameServices;
MiniDFSCluster.Builder bld = new MiniDFSCluster.Builder(conf).NumDataNodes(1).StoragesPerDatanode
(1);
if (numNameServices > 1)
{
bld.NnTopology(MiniDFSNNTopology.SimpleFederatedTopology(numNameServices));
}
cluster = bld.Build();
cluster.WaitActive();
dfs = new DistributedFileSystem[numNameServices];
for (int i = 0; i < numNameServices; i++)
{
dfs[i] = cluster.GetFileSystem(i);
}
bpids = new string[numNameServices];
for (int i_1 = 0; i_1 < numNameServices; i_1++)
{
bpids[i_1] = cluster.GetNamesystem(i_1).GetBlockPoolId();
}
datanode = cluster.GetDataNodes()[0];
blockScanner = datanode.GetBlockScanner();
for (int i_2 = 0; i_2 < numNameServices; i_2++)
{
dfs[i_2].Mkdirs(new Path("/test"));
}
data = datanode.GetFSDataset();
volumes = data.GetVolumes();
}
public virtual void TestAddVolumeFailures()
{
StartDFSCluster(1, 1);
string dataDir = cluster.GetDataDirectory();
DataNode dn = cluster.GetDataNodes()[0];
IList <string> newDirs = Lists.NewArrayList();
int NumNewDirs = 4;
for (int i = 0; i < NumNewDirs; i++)
{
FilePath newVolume = new FilePath(dataDir, "new_vol" + i);
newDirs.AddItem(newVolume.ToString());
if (i % 2 == 0)
{
// Make addVolume() fail.
newVolume.CreateNewFile();
}
}
string newValue = dn.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey) + "," + Joiner
.On(",").Join(newDirs);
try
{
dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, newValue);
NUnit.Framework.Assert.Fail("Expect to throw IOException.");
}
catch (ReconfigurationException e)
{
string errorMessage = e.InnerException.Message;
string[] messages = errorMessage.Split("\\r?\\n");
NUnit.Framework.Assert.AreEqual(2, messages.Length);
Assert.AssertThat(messages[0], CoreMatchers.ContainsString("new_vol0"));
Assert.AssertThat(messages[1], CoreMatchers.ContainsString("new_vol2"));
}
// Make sure that vol0 and vol2's metadata are not left in memory.
FsDatasetSpi <object> dataset = dn.GetFSDataset();
foreach (FsVolumeSpi volume in dataset.GetVolumes())
{
Assert.AssertThat(volume.GetBasePath(), IS.Is(CoreMatchers.Not(CoreMatchers.AnyOf
(IS.Is(newDirs[0]), IS.Is(newDirs[2])))));
}
DataStorage storage = dn.GetStorage();
for (int i_1 = 0; i_1 < storage.GetNumStorageDirs(); i_1++)
{
Storage.StorageDirectory sd = storage.GetStorageDir(i_1);
Assert.AssertThat(sd.GetRoot().ToString(), IS.Is(CoreMatchers.Not(CoreMatchers.AnyOf
(IS.Is(newDirs[0]), IS.Is(newDirs[2])))));
}
// The newly effective conf does not have vol0 and vol2.
string[] effectiveVolumes = dn.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey)
.Split(",");
NUnit.Framework.Assert.AreEqual(4, effectiveVolumes.Length);
foreach (string ev in effectiveVolumes)
{
Assert.AssertThat(StorageLocation.Parse(ev).GetFile().GetCanonicalPath(), IS.Is(CoreMatchers.Not
(CoreMatchers.AnyOf(IS.Is(newDirs[0]), IS.Is(newDirs[2])))));
}
}
/// <summary>Is the given volume still valid in the dataset?</summary>
private static bool IsValid <_T0>(FsDatasetSpi <_T0> dataset, FsVolumeSpi volume)
where _T0 : FsVolumeSpi
{
foreach (FsVolumeSpi vol in dataset.GetVolumes())
{
if (vol == volume)
{
return(true);
}
}
return(false);
}
/// <summary>
/// Test that DataStorage and BlockPoolSliceStorage remove the failed volume
/// after failure.
/// </summary>
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.TimeoutException"/>
public virtual void TestFailedVolumeBeingRemovedFromDataNode()
{
Path file1 = new Path("/test1");
DFSTestUtil.CreateFile(fs, file1, 1024, (short)2, 1L);
DFSTestUtil.WaitReplication(fs, file1, (short)2);
FilePath dn0Vol1 = new FilePath(dataDir, "data" + (2 * 0 + 1));
DataNodeTestUtils.InjectDataDirFailure(dn0Vol1);
DataNode dn0 = cluster.GetDataNodes()[0];
long lastDiskErrorCheck = dn0.GetLastDiskErrorCheck();
dn0.CheckDiskErrorAsync();
// Wait checkDiskError thread finish to discover volume failure.
while (dn0.GetLastDiskErrorCheck() == lastDiskErrorCheck)
{
Sharpen.Thread.Sleep(100);
}
// Verify dn0Vol1 has been completely removed from DN0.
// 1. dn0Vol1 is removed from DataStorage.
DataStorage storage = dn0.GetStorage();
NUnit.Framework.Assert.AreEqual(1, storage.GetNumStorageDirs());
for (int i = 0; i < storage.GetNumStorageDirs(); i++)
{
Storage.StorageDirectory sd = storage.GetStorageDir(i);
NUnit.Framework.Assert.IsFalse(sd.GetRoot().GetAbsolutePath().StartsWith(dn0Vol1.
GetAbsolutePath()));
}
string bpid = cluster.GetNamesystem().GetBlockPoolId();
BlockPoolSliceStorage bpsStorage = storage.GetBPStorage(bpid);
NUnit.Framework.Assert.AreEqual(1, bpsStorage.GetNumStorageDirs());
for (int i_1 = 0; i_1 < bpsStorage.GetNumStorageDirs(); i_1++)
{
Storage.StorageDirectory sd = bpsStorage.GetStorageDir(i_1);
NUnit.Framework.Assert.IsFalse(sd.GetRoot().GetAbsolutePath().StartsWith(dn0Vol1.
GetAbsolutePath()));
}
// 2. dn0Vol1 is removed from FsDataset
FsDatasetSpi <FsVolumeSpi> data = dn0.GetFSDataset();
foreach (FsVolumeSpi volume in data.GetVolumes())
{
Assert.AssertNotEquals(new FilePath(volume.GetBasePath()).GetAbsoluteFile(), dn0Vol1
.GetAbsoluteFile());
}
// 3. all blocks on dn0Vol1 have been removed.
foreach (ReplicaInfo replica in FsDatasetTestUtil.GetReplicas(data, bpid))
{
NUnit.Framework.Assert.IsNotNull(replica.GetVolume());
Assert.AssertNotEquals(new FilePath(replica.GetVolume().GetBasePath()).GetAbsoluteFile
(), dn0Vol1.GetAbsoluteFile());
}
// 4. dn0Vol1 is not in DN0's configuration and dataDirs anymore.
string[] dataDirStrs = dn0.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey).Split
(",");
NUnit.Framework.Assert.AreEqual(1, dataDirStrs.Length);
NUnit.Framework.Assert.IsFalse(dataDirStrs[0].Contains(dn0Vol1.GetAbsolutePath())
);
}