/// <exception cref="System.IO.IOException"/> internal virtual TestStorageMover.Replication VerifyReplication(Path file, int expectedDiskCount , int expectedArchiveCount) { TestStorageMover.Replication r = new TestStorageMover.Replication(); r.disk = expectedDiskCount; r.archive = expectedArchiveCount; return(this.GetOrVerifyReplication(file, r)); }
public virtual void TestNoSpaceArchive() { Log.Info("testNoSpaceArchive"); TestStorageMover.PathPolicyMap pathPolicyMap = new TestStorageMover.PathPolicyMap (0); TestStorageMover.NamespaceScheme nsScheme = pathPolicyMap.NewNamespaceScheme(); TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme (DefaultConf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null); TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme , nsScheme); try { test.RunBasicTest(false); // create 2 hot files with replication 3 short replication = 3; for (int i = 0; i < 2; i++) { Path p = new Path(pathPolicyMap.cold, "file" + i); DFSTestUtil.CreateFile(test.dfs, p, BlockSize, replication, 0L); WaitForAllReplicas(replication, p, test.dfs); } // set all the ARCHIVE volume to full foreach (DataNode dn in test.cluster.GetDataNodes()) { SetVolumeFull(dn, StorageType.Archive); DataNodeTestUtils.TriggerHeartbeat(dn); } { // test increasing replication but new replicas cannot be created // since no more ARCHIVE space. Path file0 = new Path(pathPolicyMap.cold, "file0"); TestStorageMover.Replication r = test.GetReplication(file0); NUnit.Framework.Assert.AreEqual(0, r.disk); short newReplication = (short)5; test.dfs.SetReplication(file0, newReplication); Sharpen.Thread.Sleep(10000); test.VerifyReplication(file0, 0, r.archive); } { // test creating a hot file Path p = new Path(pathPolicyMap.hot, "foo"); DFSTestUtil.CreateFile(test.dfs, p, BlockSize, (short)3, 0L); } { //test move a cold file to warm Path file1 = new Path(pathPolicyMap.cold, "file1"); test.dfs.Rename(file1, pathPolicyMap.warm); test.Migrate(); test.Verify(true); } } finally { test.ShutdownCluster(); } }
public virtual void TestNoSpaceDisk() { Log.Info("testNoSpaceDisk"); TestStorageMover.PathPolicyMap pathPolicyMap = new TestStorageMover.PathPolicyMap (0); TestStorageMover.NamespaceScheme nsScheme = pathPolicyMap.NewNamespaceScheme(); Configuration conf = new Configuration(DefaultConf); TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme (conf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null); TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme , nsScheme); try { test.RunBasicTest(false); // create 2 hot files with replication 3 short replication = 3; for (int i = 0; i < 2; i++) { Path p = new Path(pathPolicyMap.hot, "file" + i); DFSTestUtil.CreateFile(test.dfs, p, BlockSize, replication, 0L); WaitForAllReplicas(replication, p, test.dfs); } // set all the DISK volume to full foreach (DataNode dn in test.cluster.GetDataNodes()) { SetVolumeFull(dn, StorageType.Disk); DataNodeTestUtils.TriggerHeartbeat(dn); } // test increasing replication. Since DISK is full, // new replicas should be stored in ARCHIVE as a fallback storage. Path file0 = new Path(pathPolicyMap.hot, "file0"); TestStorageMover.Replication r = test.GetReplication(file0); short newReplication = (short)5; test.dfs.SetReplication(file0, newReplication); Sharpen.Thread.Sleep(10000); test.VerifyReplication(file0, r.disk, newReplication - r.disk); // test creating a cold file and then increase replication Path p_1 = new Path(pathPolicyMap.cold, "foo"); DFSTestUtil.CreateFile(test.dfs, p_1, BlockSize, replication, 0L); test.VerifyReplication(p_1, 0, replication); test.dfs.SetReplication(p_1, newReplication); Sharpen.Thread.Sleep(10000); test.VerifyReplication(p_1, 0, newReplication); //test move a hot file to warm Path file1 = new Path(pathPolicyMap.hot, "file1"); test.dfs.Rename(file1, pathPolicyMap.warm); test.Migrate(); test.VerifyFile(new Path(pathPolicyMap.warm, "file1"), Warm.GetId()); } finally { test.ShutdownCluster(); } }
public override bool Equals(object obj) { if (obj == this) { return(true); } else { if (obj == null || !(obj is TestStorageMover.Replication)) { return(false); } } TestStorageMover.Replication that = (TestStorageMover.Replication)obj; return(this.disk == that.disk && this.archive == that.archive); }
/// <exception cref="System.IO.IOException"/> private TestStorageMover.Replication GetOrVerifyReplication(Path file, TestStorageMover.Replication expected) { IList <LocatedBlock> lbs = this.dfs.GetClient().GetLocatedBlocks(file.ToString(), 0).GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, lbs.Count); LocatedBlock lb = lbs[0]; StringBuilder types = new StringBuilder(); TestStorageMover.Replication r = new TestStorageMover.Replication(); foreach (StorageType t in lb.GetStorageTypes()) { types.Append(t).Append(", "); if (t == StorageType.Disk) { r.disk++; } else { if (t == StorageType.Archive) { r.archive++; } else { NUnit.Framework.Assert.Fail("Unexpected storage type " + t); } } } if (expected != null) { string s = "file = " + file + "\n types = [" + types + "]"; NUnit.Framework.Assert.AreEqual(s, expected, r); } return(r); }