public virtual void TestAddVolumeFailures()
{
StartDFSCluster(1, 1);
string dataDir = cluster.GetDataDirectory();
DataNode dn = cluster.GetDataNodes()[0];
IList <string> newDirs = Lists.NewArrayList();
int NumNewDirs = 4;
for (int i = 0; i < NumNewDirs; i++)
{
FilePath newVolume = new FilePath(dataDir, "new_vol" + i);
newDirs.AddItem(newVolume.ToString());
if (i % 2 == 0)
{
// Make addVolume() fail.
newVolume.CreateNewFile();
}
}
string newValue = dn.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey) + "," + Joiner
.On(",").Join(newDirs);
try
{
dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, newValue);
NUnit.Framework.Assert.Fail("Expect to throw IOException.");
}
catch (ReconfigurationException e)
{
string errorMessage = e.InnerException.Message;
string[] messages = errorMessage.Split("\\r?\\n");
NUnit.Framework.Assert.AreEqual(2, messages.Length);
Assert.AssertThat(messages[0], CoreMatchers.ContainsString("new_vol0"));
Assert.AssertThat(messages[1], CoreMatchers.ContainsString("new_vol2"));
}
// Make sure that vol0 and vol2's metadata are not left in memory.
FsDatasetSpi <object> dataset = dn.GetFSDataset();
foreach (FsVolumeSpi volume in dataset.GetVolumes())
{
Assert.AssertThat(volume.GetBasePath(), IS.Is(CoreMatchers.Not(CoreMatchers.AnyOf
(IS.Is(newDirs[0]), IS.Is(newDirs[2])))));
}
DataStorage storage = dn.GetStorage();
for (int i_1 = 0; i_1 < storage.GetNumStorageDirs(); i_1++)
{
Storage.StorageDirectory sd = storage.GetStorageDir(i_1);
Assert.AssertThat(sd.GetRoot().ToString(), IS.Is(CoreMatchers.Not(CoreMatchers.AnyOf
(IS.Is(newDirs[0]), IS.Is(newDirs[2])))));
}
// The newly effective conf does not have vol0 and vol2.
string[] effectiveVolumes = dn.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey)
.Split(",");
NUnit.Framework.Assert.AreEqual(4, effectiveVolumes.Length);
foreach (string ev in effectiveVolumes)
{
Assert.AssertThat(StorageLocation.Parse(ev).GetFile().GetCanonicalPath(), IS.Is(CoreMatchers.Not
(CoreMatchers.AnyOf(IS.Is(newDirs[0]), IS.Is(newDirs[2])))));
}
}
public virtual void TestBlockHasMultipleReplicasOnSameDN()
{
string filename = MakeFileName(GenericTestUtils.GetMethodName());
Path filePath = new Path(filename);
// Write out a file with a few blocks.
DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize,
NumDatanodes, seed);
// Get the block list for the file with the block locations.
LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize
* NumBlocks);
// Generate a fake block report from one of the DataNodes, such
// that it reports one copy of each block on either storage.
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid);
StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode
()];
AList <Replica> blocks = new AList <Replica>();
foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks())
{
Block localBlock = locatedBlock.GetBlock().GetLocalBlock();
blocks.AddItem(new FinalizedReplica(localBlock, null, null));
}
BlockListAsLongs bll = BlockListAsLongs.Encode(blocks);
for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i)
{
FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i];
DatanodeStorage dns = new DatanodeStorage(v.GetStorageID());
reports[i] = new StorageBlockReport(dns, bll);
}
// Should not assert!
cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext
(1, 0, Runtime.NanoTime()));
// Get the block locations once again.
locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks);
// Make sure that each block has two replicas, one on each DataNode.
foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks())
{
DatanodeInfo[] locations = locatedBlock_1.GetLocations();
Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes));
Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid
()));
}
}
public virtual void Setup()
{
conf = new HdfsConfiguration();
SimulatedFSDataset.SetFactory(conf);
Configuration[] overlays = new Configuration[NumDatanodes];
for (int i = 0; i < overlays.Length; i++)
{
overlays[i] = new Configuration();
if (i == RoNodeIndex)
{
overlays[i].SetEnum(SimulatedFSDataset.ConfigPropertyState, i == RoNodeIndex ? DatanodeStorage.State
.ReadOnlyShared : DatanodeStorage.State.Normal);
}
}
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDatanodes).DataNodeConfOverlays
(overlays).Build();
fs = cluster.GetFileSystem();
blockManager = cluster.GetNameNode().GetNamesystem().GetBlockManager();
datanodeManager = blockManager.GetDatanodeManager();
client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), cluster
.GetConfiguration(0));
for (int i_1 = 0; i_1 < NumDatanodes; i_1++)
{
DataNode dataNode = cluster.GetDataNodes()[i_1];
ValidateStorageState(BlockManagerTestUtil.GetStorageReportsForDatanode(datanodeManager
.GetDatanode(dataNode.GetDatanodeId())), i_1 == RoNodeIndex ? DatanodeStorage.State
.ReadOnlyShared : DatanodeStorage.State.Normal);
}
// Create a 1 block file
DFSTestUtil.CreateFile(fs, Path, BlockSize, BlockSize, BlockSize, (short)1, seed);
LocatedBlock locatedBlock = GetLocatedBlock();
extendedBlock = locatedBlock.GetBlock();
block = extendedBlock.GetLocalBlock();
Assert.AssertThat(locatedBlock.GetLocations().Length, CoreMatchers.Is(1));
normalDataNode = locatedBlock.GetLocations()[0];
readOnlyDataNode = datanodeManager.GetDatanode(cluster.GetDataNodes()[RoNodeIndex
].GetDatanodeId());
Assert.AssertThat(normalDataNode, CoreMatchers.Is(CoreMatchers.Not(readOnlyDataNode
)));
ValidateNumberReplicas(1);
// Inject the block into the datanode with READ_ONLY_SHARED storage
cluster.InjectBlocks(0, RoNodeIndex, Collections.Singleton(block));
// There should now be 2 *locations* for the block
// Must wait until the NameNode has processed the block report for the injected blocks
WaitForLocations(2);
}
public virtual void TestLocalhostReverseLookup()
{
// 127.0.0.1 -> localhost reverse resolution does not happen on Windows.
Assume.AssumeTrue(!Shell.Windows);
// Make sure when config FS_DEFAULT_NAME_KEY using IP address,
// it will automatically convert it to hostname
HdfsConfiguration conf = new HdfsConfiguration();
conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://127.0.0.1:8020");
ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf);
NUnit.Framework.Assert.AreEqual(1, uris.Count);
foreach (URI uri in uris)
{
Assert.AssertThat(uri.GetHost(), CoreMatchers.Not("127.0.0.1"));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestGetReconfigureStatus()
{
ReconfigurationUtil ru = Org.Mockito.Mockito.Mock <ReconfigurationUtil>();
datanode.SetReconfigurationUtil(ru);
IList <ReconfigurationUtil.PropertyChange> changes = new AList <ReconfigurationUtil.PropertyChange
>();
FilePath newDir = new FilePath(cluster.GetDataDirectory(), "data_new");
newDir.Mkdirs();
changes.AddItem(new ReconfigurationUtil.PropertyChange(DFSConfigKeys.DfsDatanodeDataDirKey
, newDir.ToString(), datanode.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey)
));
changes.AddItem(new ReconfigurationUtil.PropertyChange("randomKey", "new123", "old456"
));
Org.Mockito.Mockito.When(ru.ParseChangedProperties(Matchers.Any <Configuration>(),
Matchers.Any <Configuration>())).ThenReturn(changes);
int port = datanode.GetIpcPort();
string address = "localhost:" + port;
Assert.AssertThat(admin.StartReconfiguration("datanode", address), CoreMatchers.Is
(0));
IList <string> outputs = null;
int count = 100;
while (count > 0)
{
outputs = GetReconfigureStatus("datanode", address);
if (!outputs.IsEmpty() && outputs[0].Contains("finished"))
{
break;
}
count--;
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.IsTrue(count > 0);
Assert.AssertThat(outputs.Count, CoreMatchers.Is(8));
// 3 (SUCCESS) + 4 (FAILED)
IList <StorageLocation> locations = DataNode.GetStorageLocations(datanode.GetConf(
));
Assert.AssertThat(locations.Count, CoreMatchers.Is(1));
Assert.AssertThat(locations[0].GetFile(), CoreMatchers.Is(newDir));
// Verify the directory is appropriately formatted.
NUnit.Framework.Assert.IsTrue(new FilePath(newDir, Storage.StorageDirCurrent).IsDirectory
());
int successOffset = outputs[1].StartsWith("SUCCESS:") ? 1 : 5;
int failedOffset = outputs[1].StartsWith("FAILED:") ? 1 : 4;
Assert.AssertThat(outputs[successOffset], CoreMatchers.ContainsString("Change property "
+ DFSConfigKeys.DfsDatanodeDataDirKey));
Assert.AssertThat(outputs[successOffset + 1], CoreMatchers.Is(CoreMatchers.AllOf(
CoreMatchers.ContainsString("From:"), CoreMatchers.ContainsString("data1"), CoreMatchers.ContainsString
("data2"))));
Assert.AssertThat(outputs[successOffset + 2], CoreMatchers.Is(CoreMatchers.Not(CoreMatchers.AnyOf
(CoreMatchers.ContainsString("data1"), CoreMatchers.ContainsString("data2")))));
Assert.AssertThat(outputs[successOffset + 2], CoreMatchers.Is(CoreMatchers.AllOf(
CoreMatchers.ContainsString("To"), CoreMatchers.ContainsString("data_new"))));
Assert.AssertThat(outputs[failedOffset], CoreMatchers.ContainsString("Change property randomKey"
));
Assert.AssertThat(outputs[failedOffset + 1], CoreMatchers.ContainsString("From: \"old456\""
));
Assert.AssertThat(outputs[failedOffset + 2], CoreMatchers.ContainsString("To: \"new123\""
));
}
/// <summary>
/// Test that we cannot read a file beyond its snapshot length
/// when accessing it via a snapshot path.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotfileLength()
{
hdfs.Mkdirs(sub);
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
int origLen = Blocksize + 1;
int toAppend = Blocksize;
FSDataInputStream fis = null;
FileStatus fileStatus = null;
// Create and write a file.
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, 0, Blocksize, Replication, Seed);
DFSTestUtil.AppendFile(hdfs, file1, origLen);
// Create a snapshot on the parent directory.
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
FileChecksum snapChksum1 = hdfs.GetFileChecksum(file1snap1);
Assert.AssertThat("file and snapshot file checksums are not equal", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
// Append to the file.
FSDataOutputStream @out = hdfs.Append(file1);
// Nothing has been appended yet. All checksums should still be equal.
Assert.AssertThat("file and snapshot checksums (open for append) are not equal",
hdfs.GetFileChecksum(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-open for append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
try
{
AppendTestUtil.Write(@out, 0, toAppend);
// Test reading from snapshot of file that is open for append
byte[] dataFromSnapshot = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot.Length, CoreMatchers.Is
(origLen));
// Verify that checksum didn't change
Assert.AssertThat("snapshot file checksum (pre-close) has changed", hdfs.GetFileChecksum
(file1), CoreMatchers.Is(snapChksum1));
Assert.AssertThat("snapshot checksum (post-append) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
}
finally
{
@out.Close();
}
Assert.AssertThat("file and snapshot file checksums (post-close) are equal", hdfs
.GetFileChecksum(file1), CoreMatchers.Not(snapChksum1));
Assert.AssertThat("snapshot file checksum (post-close) has changed", hdfs.GetFileChecksum
(file1snap1), CoreMatchers.Is(snapChksum1));
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen + toAppend));
fis = hdfs.Open(file1);
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen + toAppend));
fis.Close();
// Try to open the file via its snapshot path.
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen));
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(0, buffer, 0, buffer.Length);
Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen));
fis.Close();
byte[] dataFromSnapshot_1 = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1);
Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot_1.Length, CoreMatchers.Is
(origLen));
}