/// <exception cref="System.IO.IOException"/>
internal virtual TestStorageMover.Replication VerifyReplication(Path file, int expectedDiskCount
, int expectedArchiveCount)
{
TestStorageMover.Replication r = new TestStorageMover.Replication();
r.disk = expectedDiskCount;
r.archive = expectedArchiveCount;
return(this.GetOrVerifyReplication(file, r));
}
public virtual void TestNoSpaceArchive()
{
Log.Info("testNoSpaceArchive");
TestStorageMover.PathPolicyMap pathPolicyMap = new TestStorageMover.PathPolicyMap
(0);
TestStorageMover.NamespaceScheme nsScheme = pathPolicyMap.NewNamespaceScheme();
TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme
(DefaultConf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null);
TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme
, nsScheme);
try
{
test.RunBasicTest(false);
// create 2 hot files with replication 3
short replication = 3;
for (int i = 0; i < 2; i++)
{
Path p = new Path(pathPolicyMap.cold, "file" + i);
DFSTestUtil.CreateFile(test.dfs, p, BlockSize, replication, 0L);
WaitForAllReplicas(replication, p, test.dfs);
}
// set all the ARCHIVE volume to full
foreach (DataNode dn in test.cluster.GetDataNodes())
{
SetVolumeFull(dn, StorageType.Archive);
DataNodeTestUtils.TriggerHeartbeat(dn);
}
{
// test increasing replication but new replicas cannot be created
// since no more ARCHIVE space.
Path file0 = new Path(pathPolicyMap.cold, "file0");
TestStorageMover.Replication r = test.GetReplication(file0);
NUnit.Framework.Assert.AreEqual(0, r.disk);
short newReplication = (short)5;
test.dfs.SetReplication(file0, newReplication);
Sharpen.Thread.Sleep(10000);
test.VerifyReplication(file0, 0, r.archive);
}
{
// test creating a hot file
Path p = new Path(pathPolicyMap.hot, "foo");
DFSTestUtil.CreateFile(test.dfs, p, BlockSize, (short)3, 0L);
}
{
//test move a cold file to warm
Path file1 = new Path(pathPolicyMap.cold, "file1");
test.dfs.Rename(file1, pathPolicyMap.warm);
test.Migrate();
test.Verify(true);
}
}
finally
{
test.ShutdownCluster();
}
}
public virtual void TestNoSpaceDisk()
{
Log.Info("testNoSpaceDisk");
TestStorageMover.PathPolicyMap pathPolicyMap = new TestStorageMover.PathPolicyMap
(0);
TestStorageMover.NamespaceScheme nsScheme = pathPolicyMap.NewNamespaceScheme();
Configuration conf = new Configuration(DefaultConf);
TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme
(conf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null);
TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme
, nsScheme);
try
{
test.RunBasicTest(false);
// create 2 hot files with replication 3
short replication = 3;
for (int i = 0; i < 2; i++)
{
Path p = new Path(pathPolicyMap.hot, "file" + i);
DFSTestUtil.CreateFile(test.dfs, p, BlockSize, replication, 0L);
WaitForAllReplicas(replication, p, test.dfs);
}
// set all the DISK volume to full
foreach (DataNode dn in test.cluster.GetDataNodes())
{
SetVolumeFull(dn, StorageType.Disk);
DataNodeTestUtils.TriggerHeartbeat(dn);
}
// test increasing replication. Since DISK is full,
// new replicas should be stored in ARCHIVE as a fallback storage.
Path file0 = new Path(pathPolicyMap.hot, "file0");
TestStorageMover.Replication r = test.GetReplication(file0);
short newReplication = (short)5;
test.dfs.SetReplication(file0, newReplication);
Sharpen.Thread.Sleep(10000);
test.VerifyReplication(file0, r.disk, newReplication - r.disk);
// test creating a cold file and then increase replication
Path p_1 = new Path(pathPolicyMap.cold, "foo");
DFSTestUtil.CreateFile(test.dfs, p_1, BlockSize, replication, 0L);
test.VerifyReplication(p_1, 0, replication);
test.dfs.SetReplication(p_1, newReplication);
Sharpen.Thread.Sleep(10000);
test.VerifyReplication(p_1, 0, newReplication);
//test move a hot file to warm
Path file1 = new Path(pathPolicyMap.hot, "file1");
test.dfs.Rename(file1, pathPolicyMap.warm);
test.Migrate();
test.VerifyFile(new Path(pathPolicyMap.warm, "file1"), Warm.GetId());
}
finally
{
test.ShutdownCluster();
}
}
public override bool Equals(object obj)
{
if (obj == this)
{
return(true);
}
else
{
if (obj == null || !(obj is TestStorageMover.Replication))
{
return(false);
}
}
TestStorageMover.Replication that = (TestStorageMover.Replication)obj;
return(this.disk == that.disk && this.archive == that.archive);
}
/// <exception cref="System.IO.IOException"/>
private TestStorageMover.Replication GetOrVerifyReplication(Path file, TestStorageMover.Replication
expected)
{
IList <LocatedBlock> lbs = this.dfs.GetClient().GetLocatedBlocks(file.ToString(),
0).GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, lbs.Count);
LocatedBlock lb = lbs[0];
StringBuilder types = new StringBuilder();
TestStorageMover.Replication r = new TestStorageMover.Replication();
foreach (StorageType t in lb.GetStorageTypes())
{
types.Append(t).Append(", ");
if (t == StorageType.Disk)
{
r.disk++;
}
else
{
if (t == StorageType.Archive)
{
r.archive++;
}
else
{
NUnit.Framework.Assert.Fail("Unexpected storage type " + t);
}
}
}
if (expected != null)
{
string s = "file = " + file + "\n types = [" + types + "]";
NUnit.Framework.Assert.AreEqual(s, expected, r);
}
return(r);
}
