public virtual void TestNoSpaceDisk()
{
Log.Info("testNoSpaceDisk");
TestStorageMover.PathPolicyMap pathPolicyMap = new TestStorageMover.PathPolicyMap
(0);
TestStorageMover.NamespaceScheme nsScheme = pathPolicyMap.NewNamespaceScheme();
Configuration conf = new Configuration(DefaultConf);
TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme
(conf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null);
TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme
, nsScheme);
try
{
test.RunBasicTest(false);
// create 2 hot files with replication 3
short replication = 3;
for (int i = 0; i < 2; i++)
{
Path p = new Path(pathPolicyMap.hot, "file" + i);
DFSTestUtil.CreateFile(test.dfs, p, BlockSize, replication, 0L);
WaitForAllReplicas(replication, p, test.dfs);
}
// set all the DISK volume to full
foreach (DataNode dn in test.cluster.GetDataNodes())
{
SetVolumeFull(dn, StorageType.Disk);
DataNodeTestUtils.TriggerHeartbeat(dn);
}
// test increasing replication. Since DISK is full,
// new replicas should be stored in ARCHIVE as a fallback storage.
Path file0 = new Path(pathPolicyMap.hot, "file0");
TestStorageMover.Replication r = test.GetReplication(file0);
short newReplication = (short)5;
test.dfs.SetReplication(file0, newReplication);
Sharpen.Thread.Sleep(10000);
test.VerifyReplication(file0, r.disk, newReplication - r.disk);
// test creating a cold file and then increase replication
Path p_1 = new Path(pathPolicyMap.cold, "foo");
DFSTestUtil.CreateFile(test.dfs, p_1, BlockSize, replication, 0L);
test.VerifyReplication(p_1, 0, replication);
test.dfs.SetReplication(p_1, newReplication);
Sharpen.Thread.Sleep(10000);
test.VerifyReplication(p_1, 0, newReplication);
//test move a hot file to warm
Path file1 = new Path(pathPolicyMap.hot, "file1");
test.dfs.Rename(file1, pathPolicyMap.warm);
test.Migrate();
test.VerifyFile(new Path(pathPolicyMap.warm, "file1"), Warm.GetId());
}
finally
{
test.ShutdownCluster();
}
}
/// <exception cref="System.IO.IOException"/>
internal static HdfsFileStatus SetStoragePolicy(FSDirectory fsd, BlockManager bm,
string src, string policyName)
{
if (!fsd.IsStoragePolicyEnabled())
{
throw new IOException("Failed to set storage policy since " + DFSConfigKeys.DfsStoragePolicyEnabledKey
+ " is set to false.");
}
FSPermissionChecker pc = fsd.GetPermissionChecker();
byte[][] pathComponents = FSDirectory.GetPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.WriteLock();
try
{
src = FSDirectory.ResolvePath(src, pathComponents, fsd);
iip = fsd.GetINodesInPath4Write(src);
if (fsd.IsPermissionEnabled())
{
fsd.CheckPathAccess(pc, iip, FsAction.Write);
}
// get the corresponding policy and make sure the policy name is valid
BlockStoragePolicy policy = bm.GetStoragePolicy(policyName);
if (policy == null)
{
throw new HadoopIllegalArgumentException("Cannot find a block policy with the name "
+ policyName);
}
UnprotectedSetStoragePolicy(fsd, bm, iip, policy.GetId());
fsd.GetEditLog().LogSetStoragePolicy(src, policy.GetId());
}
finally
{
fsd.WriteUnlock();
}
return(fsd.GetAuditFileInfo(iip));
}
/// <exception cref="System.Exception"/>
private void VerifyFile(Path parent, HdfsFileStatus status, byte expectedPolicyId
)
{
HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus)status;
byte policyId = fileStatus.GetStoragePolicy();
BlockStoragePolicy policy = this.policies.GetPolicy(policyId);
if (expectedPolicyId != null)
{
NUnit.Framework.Assert.AreEqual(unchecked ((byte)expectedPolicyId), policy.GetId()
);
}
IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication());
foreach (LocatedBlock lb in fileStatus.GetBlockLocations().GetLocatedBlocks())
{
Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
));
NUnit.Framework.Assert.IsTrue(fileStatus.GetFullName(parent.ToString()) + " with policy "
+ policy + " has non-empty overlap: " + diff + ", the corresponding block is "
+ lb.GetBlock().GetLocalBlock(), diff.RemoveOverlap(true));
}
}
