public virtual void TestAbandonBlock() { string src = FileNamePrefix + "foo"; // Start writing a file but do not close it FSDataOutputStream fout = fs.Create(new Path(src), true, 4096, (short)1, 512L); for (int i = 0; i < 1024; i++) { fout.Write(123); } fout.Hflush(); long fileId = ((DFSOutputStream)fout.GetWrappedStream()).GetFileId(); // Now abandon the last block DFSClient dfsclient = DFSClientAdapter.GetDFSClient(fs); LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue ); int orginalNumBlocks = blocks.LocatedBlockCount(); LocatedBlock b = blocks.GetLastLocatedBlock(); dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName ); // call abandonBlock again to make sure the operation is idempotent dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName ); // And close the file fout.Close(); // Close cluster and check the block has been abandoned after restart cluster.RestartNameNode(); blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue); NUnit.Framework.Assert.AreEqual("Blocks " + b + " has not been abandoned.", orginalNumBlocks , blocks.LocatedBlockCount() + 1); }
/// <exception cref="System.IO.IOException"/> private void TestHSyncOperation(bool testWithAppend) { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); DistributedFileSystem fs = cluster.GetFileSystem(); Path p = new Path("/testHSync/foo"); int len = 1 << 16; FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag .Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null); if (testWithAppend) { // re-open the file with append call @out.Close(); @out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.SyncBlock), 4096, null ); } @out.Hflush(); // hflush does not sync CheckSyncMetric(cluster, 0); @out.Hsync(); // hsync on empty file does nothing CheckSyncMetric(cluster, 0); @out.Write(1); CheckSyncMetric(cluster, 0); @out.Hsync(); CheckSyncMetric(cluster, 1); // avoiding repeated hsyncs is a potential future optimization @out.Hsync(); CheckSyncMetric(cluster, 2); @out.Hflush(); // hflush still does not sync CheckSyncMetric(cluster, 2); @out.Close(); // close is sync'ing CheckSyncMetric(cluster, 3); // same with a file created with out SYNC_BLOCK @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag.Create, CreateFlag .Overwrite), 4096, (short)1, len, null); @out.Hsync(); CheckSyncMetric(cluster, 3); @out.Write(1); CheckSyncMetric(cluster, 3); @out.Hsync(); CheckSyncMetric(cluster, 4); // repeated hsyncs @out.Hsync(); CheckSyncMetric(cluster, 5); @out.Close(); // close does not sync (not opened with SYNC_BLOCK) CheckSyncMetric(cluster, 5); cluster.Shutdown(); }
/// <exception cref="System.IO.IOException"/> private void TestPersistHelper(Configuration conf) { MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); FSNamesystem fsn = cluster.GetNamesystem(); DistributedFileSystem fs = cluster.GetFileSystem(); Path dir = new Path("/abc/def"); Path file1 = new Path(dir, "f1"); Path file2 = new Path(dir, "f2"); // create an empty file f1 fs.Create(file1).Close(); // create an under-construction file f2 FSDataOutputStream @out = fs.Create(file2); @out.WriteBytes("hello"); ((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .UpdateLength)); // checkpoint fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); fs.SaveNamespace(); fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.RestartNameNode(); cluster.WaitActive(); fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(fs.IsDirectory(dir)); NUnit.Framework.Assert.IsTrue(fs.Exists(file1)); NUnit.Framework.Assert.IsTrue(fs.Exists(file2)); // check internals of file2 INodeFile file2Node = fsn.dir.GetINode4Write(file2.ToString()).AsFile(); NUnit.Framework.Assert.AreEqual("hello".Length, file2Node.ComputeFileSize()); NUnit.Framework.Assert.IsTrue(file2Node.IsUnderConstruction()); BlockInfoContiguous[] blks = file2Node.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Length); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.UnderConstruction , blks[0].GetBlockUCState()); // check lease manager LeaseManager.Lease lease = fsn.leaseManager.GetLeaseByPath(file2.ToString()); NUnit.Framework.Assert.IsNotNull(lease); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Test the updation of NeededReplications for the Appended Block</summary> /// <exception cref="System.Exception"/> public virtual void TestUpdateNeededReplicationsForAppendedFile() { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); DistributedFileSystem fileSystem = null; try { // create a file. fileSystem = cluster.GetFileSystem(); Path f = new Path("/testAppend"); FSDataOutputStream create = fileSystem.Create(f, (short)2); create.Write(Sharpen.Runtime.GetBytesForString("/testAppend")); create.Close(); // Append to the file. FSDataOutputStream append = fileSystem.Append(f); append.Write(Sharpen.Runtime.GetBytesForString("/testAppend")); append.Close(); // Start a new datanode cluster.StartDataNodes(conf, 1, true, null, null); // Check for replications DFSTestUtil.WaitReplication(fileSystem, f, (short)2); } finally { if (null != fileSystem) { fileSystem.Close(); } cluster.Shutdown(); } }
/// <summary> /// The idea for making sure that there is no more than one instance /// running in an HDFS is to create a file in the HDFS, writes the hostname /// of the machine on which the instance is running to the file, but did not /// close the file until it exits. /// </summary> /// <remarks> /// The idea for making sure that there is no more than one instance /// running in an HDFS is to create a file in the HDFS, writes the hostname /// of the machine on which the instance is running to the file, but did not /// close the file until it exits. /// This prevents the second instance from running because it can not /// creates the file while the first one is running. /// This method checks if there is any running instance. If no, mark yes. /// Note that this is an atomic operation. /// </remarks> /// <returns> /// null if there is a running instance; /// otherwise, the output stream to the newly created file. /// </returns> /// <exception cref="System.IO.IOException"/> private OutputStream CheckAndMarkRunning() { try { if (fs.Exists(idPath)) { // try appending to it so that it will fail fast if another balancer is // running. IOUtils.CloseStream(fs.Append(idPath)); fs.Delete(idPath, true); } FSDataOutputStream fsout = fs.Create(idPath, false); // mark balancer idPath to be deleted during filesystem closure fs.DeleteOnExit(idPath); if (write2IdFile) { fsout.WriteBytes(Sharpen.Runtime.GetLocalHost().GetHostName()); fsout.Hflush(); } return(fsout); } catch (RemoteException e) { if (typeof(AlreadyBeingCreatedException).FullName.Equals(e.GetClassName())) { return(null); } else { throw; } } }
public virtual void TestSaveNamespaceWithRenamedLease() { MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration()).NumDataNodes (1).Build(); cluster.WaitActive(); DistributedFileSystem fs = (DistributedFileSystem)cluster.GetFileSystem(); OutputStream @out = null; try { fs.Mkdirs(new Path("/test-target")); @out = fs.Create(new Path("/test-source/foo")); // don't close fs.Rename(new Path("/test-source/"), new Path("/test-target/")); fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); cluster.GetNameNodeRpc().SaveNamespace(); fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); } finally { IOUtils.Cleanup(Log, @out, fs); if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> protected internal void MakeTestFile(Path path, long length, bool isLazyPersist) { EnumSet <CreateFlag> createFlags = EnumSet.Of(CreateFlag.Create); if (isLazyPersist) { createFlags.AddItem(CreateFlag.LazyPersist); } FSDataOutputStream fos = null; try { fos = fs.Create(path, FsPermission.GetFileDefault(), createFlags, BufferLength, ReplFactor , BlockSize, null); // Allocate a block. byte[] buffer = new byte[BufferLength]; for (int bytesWritten = 0; bytesWritten < length;) { fos.Write(buffer, 0, buffer.Length); bytesWritten += buffer.Length; } if (length > 0) { fos.Hsync(); } } finally { IOUtils.CloseQuietly(fos); } }
/// <summary> /// Test fsimage loading when 1) there is an empty file loaded from fsimage, /// and 2) there is later an append operation to be applied from edit log. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestLoadImageWithEmptyFile() { // create an empty file Path file = new Path(dir, "file"); FSDataOutputStream @out = hdfs.Create(file); @out.Close(); // save namespace hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); hdfs.SaveNamespace(); hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); // append to the empty file @out = hdfs.Append(file); @out.Write(1); @out.Close(); // restart cluster cluster.Shutdown(); cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication ).Build(); cluster.WaitActive(); hdfs = cluster.GetFileSystem(); FileStatus status = hdfs.GetFileStatus(file); NUnit.Framework.Assert.AreEqual(1, status.GetLen()); }
/// <exception cref="System.IO.IOException"/> internal SlowWriter(DistributedFileSystem fs, Path filepath, long sleepms) : base(typeof(TestReplaceDatanodeOnFailure.SlowWriter).Name + ":" + filepath) { this.filepath = filepath; this.@out = (HdfsDataOutputStream)fs.Create(filepath, Replication); this.sleepms = sleepms; }
public virtual void TestMoverFailedRetry() { // HDFS-8147 Configuration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsMoverRetryMaxAttemptsKey, "2"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).StorageTypes (new StorageType[][] { new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType [] { StorageType.Disk, StorageType.Archive } }).Build(); try { cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); string file = "/testMoverFailedRetry"; // write to DISK FSDataOutputStream @out = dfs.Create(new Path(file), (short)2); @out.WriteChars("testMoverFailedRetry"); @out.Close(); // Delete block file so, block move will fail with FileNotFoundException LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0); cluster.CorruptBlockOnDataNodesByDeletingBlockFile(lb.GetBlock()); // move to ARCHIVE dfs.SetStoragePolicy(new Path(file), "COLD"); int rc = ToolRunner.Run(conf, new Mover.Cli(), new string[] { "-p", file.ToString () }); NUnit.Framework.Assert.AreEqual("Movement should fail after some retry", ExitStatus .IoException.GetExitCode(), rc); } finally { cluster.Shutdown(); } }
public virtual void TestCloseTwice() { DistributedFileSystem fs = cluster.GetFileSystem(); FSDataOutputStream os = fs.Create(new Path("/test")); DFSOutputStream dos = (DFSOutputStream)Whitebox.GetInternalState(os, "wrappedStream" ); AtomicReference <IOException> ex = (AtomicReference <IOException>)Whitebox.GetInternalState (dos, "lastException"); NUnit.Framework.Assert.AreEqual(null, ex.Get()); dos.Close(); IOException dummy = new IOException("dummy"); ex.Set(dummy); try { dos.Close(); } catch (IOException e) { NUnit.Framework.Assert.AreEqual(e, dummy); } NUnit.Framework.Assert.AreEqual(null, ex.Get()); dos.Close(); }
public virtual void TestAppendWithPipelineRecovery() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; FSDataOutputStream @out = null; try { cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(true).ManageNameDfsDirs (true).NumDataNodes(4).Racks(new string[] { "/rack1", "/rack1", "/rack2", "/rack2" }).Build(); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); Path path = new Path("/test1"); @out = fs.Create(path, true, BlockSize, (short)3, BlockSize); AppendTestUtil.Write(@out, 0, 1024); @out.Close(); cluster.StopDataNode(3); @out = fs.Append(path); AppendTestUtil.Write(@out, 1024, 1024); @out.Close(); cluster.RestartNameNode(true); AppendTestUtil.Check(fs, path, 2048); } finally { IOUtils.CloseStream(@out); if (null != cluster) { cluster.Shutdown(); } } }
public virtual void TestAppendLessThanChecksumChunk() { byte[] buf = new byte[1024]; MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NumDataNodes (1).Build(); cluster.WaitActive(); try { using (DistributedFileSystem fs = cluster.GetFileSystem()) { int len1 = 200; int len2 = 300; Path p = new Path("/foo"); FSDataOutputStream @out = fs.Create(p); @out.Write(buf, 0, len1); @out.Close(); @out = fs.Append(p); @out.Write(buf, 0, len2); // flush but leave open @out.Hflush(); // read data to verify the replica's content and checksum are correct FSDataInputStream @in = fs.Open(p); int length = @in.Read(0, buf, 0, len1 + len2); NUnit.Framework.Assert.IsTrue(length > 0); @in.Close(); @out.Close(); } } finally { cluster.Shutdown(); } }
public virtual void TestHedgedReadLoopTooManyTimes() { Configuration conf = new Configuration(); int numHedgedReadPoolThreads = 5; int hedgedReadTimeoutMillis = 50; conf.SetInt(DFSConfigKeys.DfsDfsclientHedgedReadThreadpoolSize, numHedgedReadPoolThreads ); conf.SetLong(DFSConfigKeys.DfsDfsclientHedgedReadThresholdMillis, hedgedReadTimeoutMillis ); conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 0); // Set up the InjectionHandler DFSClientFaultInjector.instance = Org.Mockito.Mockito.Mock <DFSClientFaultInjector >(); DFSClientFaultInjector injector = DFSClientFaultInjector.instance; int sleepMs = 100; Org.Mockito.Mockito.DoAnswer(new _Answer_296(hedgedReadTimeoutMillis, sleepMs)).When (injector).FetchFromDatanodeException(); Org.Mockito.Mockito.DoAnswer(new _Answer_309(sleepMs)).When(injector).ReadFromDatanodeDelay (); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format( true).Build(); DistributedFileSystem fileSys = cluster.GetFileSystem(); DFSClient dfsClient = fileSys.GetClient(); FSDataOutputStream output = null; DFSInputStream input = null; string filename = "/hedgedReadMaxOut.dat"; try { Path file = new Path(filename); output = fileSys.Create(file, (short)2); byte[] data = new byte[64 * 1024]; output.Write(data); output.Flush(); output.Write(data); output.Flush(); output.Write(data); output.Flush(); output.Close(); byte[] buffer = new byte[64 * 1024]; input = dfsClient.Open(filename); input.Read(0, buffer, 0, 1024); input.Close(); NUnit.Framework.Assert.AreEqual(3, input.GetHedgedReadOpsLoopNumForTesting()); } catch (BlockMissingException) { NUnit.Framework.Assert.IsTrue(false); } finally { Org.Mockito.Mockito.Reset(injector); IOUtils.Cleanup(null, input); IOUtils.Cleanup(null, output); fileSys.Close(); cluster.Shutdown(); } }
public virtual void TestOpenFilesWithRename() { Path path = new Path("/test"); DoWriteAndAbort(fs, path); // check for zero sized blocks Path fileWithEmptyBlock = new Path("/test/test/test4"); fs.Create(fileWithEmptyBlock); NamenodeProtocols nameNodeRpc = cluster.GetNameNodeRpc(); string clientName = fs.GetClient().GetClientName(); // create one empty block nameNodeRpc.AddBlock(fileWithEmptyBlock.ToString(), clientName, null, null, INodeId .GrandfatherInodeId, null); fs.CreateSnapshot(path, "s2"); fs.Rename(new Path("/test/test"), new Path("/test/test-renamed")); fs.Delete(new Path("/test/test-renamed"), true); NameNode nameNode = cluster.GetNameNode(); NameNodeAdapter.EnterSafeMode(nameNode, false); NameNodeAdapter.SaveNamespace(nameNode); NameNodeAdapter.LeaveSafeMode(nameNode); cluster.RestartNameNode(true); }
public virtual void TestAppend() { int maxOldFileLen = 2 * BlockSize + 1; int maxFlushedBytes = BlockSize; byte[] contents = AppendTestUtil.InitBuffer(maxOldFileLen + 2 * maxFlushedBytes); for (int oldFileLen = 0; oldFileLen <= maxOldFileLen; oldFileLen++) { for (int flushedBytes1 = 0; flushedBytes1 <= maxFlushedBytes; flushedBytes1++) { for (int flushedBytes2 = 0; flushedBytes2 <= maxFlushedBytes; flushedBytes2++) { int fileLen = oldFileLen + flushedBytes1 + flushedBytes2; // create the initial file of oldFileLen Path p = new Path("foo" + oldFileLen + "_" + flushedBytes1 + "_" + flushedBytes2); Log.Info("Creating file " + p); FSDataOutputStream @out = fs.Create(p, false, conf.GetInt(CommonConfigurationKeys .IoFileBufferSizeKey, 4096), Replication, BlockSize); @out.Write(contents, 0, oldFileLen); @out.Close(); // append flushedBytes bytes to the file @out = fs.Append(p); @out.Write(contents, oldFileLen, flushedBytes1); @out.Hflush(); // write another flushedBytes2 bytes to the file @out.Write(contents, oldFileLen + flushedBytes1, flushedBytes2); @out.Close(); // validate the file content AppendTestUtil.CheckFullFile(fs, p, fileLen, contents, p.ToString()); fs.Delete(p, false); } } } }
/// <exception cref="System.IO.IOException"/> private void TestXAttr(bool persistNamespace) { Path path = new Path("/p"); DistributedFileSystem fs = cluster.GetFileSystem(); fs.Create(path).Close(); fs.SetXAttr(path, name1, value1, EnumSet.Of(XAttrSetFlag.Create)); fs.SetXAttr(path, name2, value2, EnumSet.Of(XAttrSetFlag.Create)); fs.SetXAttr(path, name3, null, EnumSet.Of(XAttrSetFlag.Create)); Restart(fs, persistNamespace); IDictionary <string, byte[]> xattrs = fs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 3); Assert.AssertArrayEquals(value1, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); Assert.AssertArrayEquals(value3, xattrs[name3]); fs.SetXAttr(path, name1, newValue1, EnumSet.Of(XAttrSetFlag.Replace)); Restart(fs, persistNamespace); xattrs = fs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 3); Assert.AssertArrayEquals(newValue1, xattrs[name1]); Assert.AssertArrayEquals(value2, xattrs[name2]); Assert.AssertArrayEquals(value3, xattrs[name3]); fs.RemoveXAttr(path, name1); fs.RemoveXAttr(path, name2); fs.RemoveXAttr(path, name3); Restart(fs, persistNamespace); xattrs = fs.GetXAttrs(path); NUnit.Framework.Assert.AreEqual(xattrs.Count, 0); }
public virtual void TestScheduleSameBlock() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build(); try { cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); string file = "/testScheduleSameBlock/file"; { FSDataOutputStream @out = dfs.Create(new Path(file)); @out.WriteChars("testScheduleSameBlock"); @out.Close(); } Org.Apache.Hadoop.Hdfs.Server.Mover.Mover mover = NewMover(conf); mover.Init(); Mover.Processor processor = new Mover.Processor(this); LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0); IList <Mover.MLocation> locations = Mover.MLocation.ToLocations(lb); Mover.MLocation ml = locations[0]; Dispatcher.DBlock db = mover.NewDBlock(lb.GetBlock().GetLocalBlock(), locations); IList <StorageType> storageTypes = new AList <StorageType>(Arrays.AsList(StorageType .Default, StorageType.Default)); NUnit.Framework.Assert.IsTrue(processor.ScheduleMoveReplica(db, ml, storageTypes) ); NUnit.Framework.Assert.IsFalse(processor.ScheduleMoveReplica(db, ml, storageTypes )); } finally { cluster.Shutdown(); } }
public virtual void HSyncEndBlock_00() { int preferredBlockSize = 1024; Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, preferredBlockSize); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); DistributedFileSystem fileSystem = cluster.GetFileSystem(); FSDataOutputStream stm = null; try { Path path = new Path("/" + fName); stm = fileSystem.Create(path, true, 4096, (short)2, AppendTestUtil.BlockSize); System.Console.Out.WriteLine("Created file " + path.ToString()); ((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .EndBlock)); long currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(0L, currentFileLength); LocatedBlocks blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(0, blocks.GetLocatedBlocks().Count); // write a block and call hsync(end_block) at the block boundary stm.Write(new byte[preferredBlockSize]); ((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .EndBlock)); currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(preferredBlockSize, currentFileLength); blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(1, blocks.GetLocatedBlocks().Count); // call hsync then call hsync(end_block) immediately stm.Write(new byte[preferredBlockSize / 2]); stm.Hsync(); ((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .EndBlock)); currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2, currentFileLength ); blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(2, blocks.GetLocatedBlocks().Count); stm.Write(new byte[preferredBlockSize / 4]); stm.Hsync(); currentFileLength = fileSystem.GetFileStatus(path).GetLen(); NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2 + preferredBlockSize / 4, currentFileLength); blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0); NUnit.Framework.Assert.AreEqual(3, blocks.GetLocatedBlocks().Count); } finally { IOUtils.Cleanup(null, stm, fileSystem); if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public virtual void TestRecoverLease() { NUnit.Framework.Assert.AreEqual("ret: 1, You must supply a -path argument to recoverLease." , RunCmd(new string[] { "recoverLease", "-retries", "1" })); FSDataOutputStream @out = fs.Create(new Path("/foo")); @out.Write(123); @out.Close(); NUnit.Framework.Assert.AreEqual("ret: 0, recoverLease SUCCEEDED on /foo", RunCmd( new string[] { "recoverLease", "-path", "/foo" })); }
public virtual void TestFailedAppendBlockRejection() { Configuration conf = new HdfsConfiguration(); conf.Set("dfs.client.block.write.replace-datanode-on-failure.enable", "false"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); DistributedFileSystem fs = null; try { fs = cluster.GetFileSystem(); Path path = new Path("/test"); FSDataOutputStream @out = fs.Create(path); @out.WriteBytes("hello\n"); @out.Close(); // stop one datanode MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0); string dnAddress = dnProp.datanode.GetXferAddress().ToString(); if (dnAddress.StartsWith("/")) { dnAddress = Sharpen.Runtime.Substring(dnAddress, 1); } // append again to bump genstamps for (int i = 0; i < 2; i++) { @out = fs.Append(path); @out.WriteBytes("helloagain\n"); @out.Close(); } // re-open and make the block state as underconstruction @out = fs.Append(path); cluster.RestartDataNode(dnProp, true); // wait till the block report comes Sharpen.Thread.Sleep(2000); // check the block locations, this should not contain restarted datanode BlockLocation[] locations = fs.GetFileBlockLocations(path, 0, long.MaxValue); string[] names = locations[0].GetNames(); foreach (string node in names) { if (node.Equals(dnAddress)) { NUnit.Framework.Assert.Fail("Failed append should not be present in latest block locations." ); } } @out.Close(); } finally { IOUtils.CloseStream(fs); cluster.Shutdown(); } }
/// <summary>Test to verify the race between finalizeBlock and Lease recovery</summary> /// <exception cref="System.Exception"/> public virtual void TestRaceBetweenReplicaRecoveryAndFinalizeBlock() { TearDown(); // Stop the Mocked DN started in startup() Configuration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisKey, "1000"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); try { cluster.WaitClusterUp(); DistributedFileSystem fs = cluster.GetFileSystem(); Path path = new Path("/test"); FSDataOutputStream @out = fs.Create(path); @out.WriteBytes("data"); @out.Hsync(); IList <LocatedBlock> blocks = DFSTestUtil.GetAllBlocks(fs.Open(path)); LocatedBlock block = blocks[0]; DataNode dataNode = cluster.GetDataNodes()[0]; AtomicBoolean recoveryInitResult = new AtomicBoolean(true); Sharpen.Thread recoveryThread = new _Thread_612(block, dataNode, recoveryInitResult ); recoveryThread.Start(); try { @out.Close(); } catch (IOException e) { NUnit.Framework.Assert.IsTrue("Writing should fail", e.Message.Contains("are bad. Aborting..." )); } finally { recoveryThread.Join(); } NUnit.Framework.Assert.IsTrue("Recovery should be initiated successfully", recoveryInitResult .Get()); dataNode.UpdateReplicaUnderRecovery(block.GetBlock(), block.GetBlock().GetGenerationStamp () + 1, block.GetBlock().GetBlockId(), block.GetBlockSize()); } finally { if (null != cluster) { cluster.Shutdown(); cluster = null; } } }
public virtual void TestImmediateRecoveryOfLease() { //create a file // write bytes into the file. byte[] actual = new byte[FileSize]; int size = AppendTestUtil.NextInt(FileSize); Path filepath = CreateFile("/immediateRecoverLease-shortlease", size, true); // set the soft limit to be 1 second so that the // namenode triggers lease recovery on next attempt to write-for-open. cluster.SetLeasePeriod(ShortLeasePeriod, LongLeasePeriod); RecoverLeaseUsingCreate(filepath); VerifyFile(dfs, filepath, actual, size); //test recoverLease // set the soft limit to be 1 hour but recoverLease should // close the file immediately cluster.SetLeasePeriod(LongLeasePeriod, LongLeasePeriod); size = AppendTestUtil.NextInt(FileSize); filepath = CreateFile("/immediateRecoverLease-longlease", size, false); // test recoverLease from a different client RecoverLease(filepath, null); VerifyFile(dfs, filepath, actual, size); // test recoverlease from the same client size = AppendTestUtil.NextInt(FileSize); filepath = CreateFile("/immediateRecoverLease-sameclient", size, false); // create another file using the same client Path filepath1 = new Path(filepath.ToString() + AppendTestUtil.NextInt()); FSDataOutputStream stm = dfs.Create(filepath1, true, BufSize, ReplicationNum, BlockSize ); // recover the first file RecoverLease(filepath, dfs); VerifyFile(dfs, filepath, actual, size); // continue to write to the second file stm.Write(buffer, 0, size); stm.Close(); VerifyFile(dfs, filepath1, actual, size); }
/// <exception cref="System.IO.IOException"/> private void DoWriteAndAbort(DistributedFileSystem fs, Path path) { fs.Mkdirs(path); fs.AllowSnapshot(path); DFSTestUtil.CreateFile(fs, new Path("/test/test1"), 100, (short)2, 100024L); DFSTestUtil.CreateFile(fs, new Path("/test/test2"), 100, (short)2, 100024L); Path file = new Path("/test/test/test2"); FSDataOutputStream @out = fs.Create(file); for (int i = 0; i < 2; i++) { long count = 0; while (count < 1048576) { @out.WriteBytes("hell"); count += 4; } } ((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .UpdateLength)); DFSTestUtil.AbortStream((DFSOutputStream)@out.GetWrappedStream()); Path file2 = new Path("/test/test/test3"); FSDataOutputStream out2 = fs.Create(file2); for (int i_1 = 0; i_1 < 2; i_1++) { long count = 0; while (count < 1048576) { out2.WriteBytes("hell"); count += 4; } } ((DFSOutputStream)out2.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .UpdateLength)); DFSTestUtil.AbortStream((DFSOutputStream)out2.GetWrappedStream()); fs.CreateSnapshot(path, "s1"); }
public virtual void TestAppend() { Configuration conf = new HdfsConfiguration(); short Replication = (short)3; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); try { DistributedFileSystem fs = cluster.GetFileSystem(); Path f = new Path(Dir, "testAppend"); { Log.Info("create an empty file " + f); fs.Create(f, Replication).Close(); FileStatus status = fs.GetFileStatus(f); NUnit.Framework.Assert.AreEqual(Replication, status.GetReplication()); NUnit.Framework.Assert.AreEqual(0L, status.GetLen()); } byte[] bytes = new byte[1000]; { Log.Info("append " + bytes.Length + " bytes to " + f); FSDataOutputStream @out = fs.Append(f); @out.Write(bytes); @out.Close(); FileStatus status = fs.GetFileStatus(f); NUnit.Framework.Assert.AreEqual(Replication, status.GetReplication()); NUnit.Framework.Assert.AreEqual(bytes.Length, status.GetLen()); } { Log.Info("append another " + bytes.Length + " bytes to " + f); try { FSDataOutputStream @out = fs.Append(f); @out.Write(bytes); @out.Close(); NUnit.Framework.Assert.Fail(); } catch (IOException ioe) { Log.Info("This exception is expected", ioe); } } } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestTC1() { Path p = new Path("/TC1/foo"); System.Console.Out.WriteLine("p=" + p); //a. Create file and write one block of data. Close file. int len1 = (int)BlockSize; { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } // Reopen file to append. Append half block of data. Close file. int len2 = (int)BlockSize / 2; { FSDataOutputStream @out = fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //b. Reopen file and read 1.5 blocks worth of data. Close file. AppendTestUtil.Check(fs, p, len1 + len2); }
/// <exception cref="System.Exception"/> public virtual void TestTwoReplicaSameStorageTypeShouldNotSelect() { // HDFS-8147 Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).StorageTypes (new StorageType[][] { new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType[] { StorageType.Disk, StorageType.Disk }, new StorageType[] { StorageType.Disk, StorageType.Archive } }).Build(); try { cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); string file = "/testForTwoReplicaSameStorageTypeShouldNotSelect"; // write to DISK FSDataOutputStream @out = dfs.Create(new Path(file), (short)2); @out.WriteChars("testForTwoReplicaSameStorageTypeShouldNotSelect"); @out.Close(); // verify before movement LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0); StorageType[] storageTypes = lb.GetStorageTypes(); foreach (StorageType storageType in storageTypes) { NUnit.Framework.Assert.IsTrue(StorageType.Disk == storageType); } // move to ARCHIVE dfs.SetStoragePolicy(new Path(file), "COLD"); int rc = ToolRunner.Run(conf, new Mover.Cli(), new string[] { "-p", file.ToString () }); NUnit.Framework.Assert.AreEqual("Movement to ARCHIVE should be successfull", 0, rc ); // Wait till namenode notified Sharpen.Thread.Sleep(3000); lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0); storageTypes = lb.GetStorageTypes(); int archiveCount = 0; foreach (StorageType storageType_1 in storageTypes) { if (StorageType.Archive == storageType_1) { archiveCount++; } } NUnit.Framework.Assert.AreEqual(archiveCount, 2); } finally { cluster.Shutdown(); } }
/// <exception cref="System.Exception"/> public virtual void TestFavoredNodesEndToEnd() { //create 10 files with random preferred nodes for (int i = 0; i < NumFiles; i++) { Random rand = new Random(Runtime.CurrentTimeMillis() + i); //pass a new created rand so as to get a uniform distribution each time //without too much collisions (look at the do-while loop in getDatanodes) IPEndPoint[] datanode = GetDatanodes(rand); Path p = new Path("/filename" + i); FSDataOutputStream @out = dfs.Create(p, FsPermission.GetDefault(), true, 4096, (short )3, 4096L, null, datanode); @out.Write(SomeBytes); @out.Close(); BlockLocation[] locations = GetBlockLocations(p); //verify the files got created in the right nodes foreach (BlockLocation loc in locations) { string[] hosts = loc.GetNames(); string[] hosts1 = GetStringForInetSocketAddrs(datanode); NUnit.Framework.Assert.IsTrue(CompareNodes(hosts, hosts1)); } } }
public virtual void TestAppend2AfterSoftLimit() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); //Set small soft-limit for lease long softLimit = 1L; long hardLimit = 9999999L; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.SetLeasePeriod(softLimit, hardLimit); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); DistributedFileSystem fs2 = new DistributedFileSystem(); fs2.Initialize(fs.GetUri(), conf); Path testPath = new Path("/testAppendAfterSoftLimit"); byte[] fileContents = AppendTestUtil.InitBuffer(32); // create a new file without closing FSDataOutputStream @out = fs.Create(testPath); @out.Write(fileContents); //Wait for > soft-limit Sharpen.Thread.Sleep(250); try { FSDataOutputStream appendStream2 = fs2.Append(testPath, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null); appendStream2.Write(fileContents); appendStream2.Close(); NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen ()); // make sure we now have 1 block since the first writer was revoked LocatedBlocks blks = fs.GetClient().GetLocatedBlocks(testPath.ToString(), 0L); NUnit.Framework.Assert.AreEqual(1, blks.GetLocatedBlocks().Count); foreach (LocatedBlock blk in blks.GetLocatedBlocks()) { NUnit.Framework.Assert.AreEqual(fileContents.Length, blk.GetBlockSize()); } } finally { fs.Close(); fs2.Close(); cluster.Shutdown(); } }
public virtual void TestBlockRecoveryWithLessMetafile() { Configuration conf = new Configuration(); conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser ().GetShortUserName()); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); Path file = new Path("/testRecoveryFile"); DistributedFileSystem dfs = cluster.GetFileSystem(); FSDataOutputStream @out = dfs.Create(file); int count = 0; while (count < 2 * 1024 * 1024) { @out.WriteBytes("Data"); count += 4; } @out.Hsync(); // abort the original stream ((DFSOutputStream)@out.GetWrappedStream()).Abort(); LocatedBlocks locations = cluster.GetNameNodeRpc().GetBlockLocations(file.ToString (), 0, count); ExtendedBlock block = locations.Get(0).GetBlock(); DataNode dn = cluster.GetDataNodes()[0]; BlockLocalPathInfo localPathInfo = dn.GetBlockLocalPathInfo(block, null); FilePath metafile = new FilePath(localPathInfo.GetMetaPath()); NUnit.Framework.Assert.IsTrue(metafile.Exists()); // reduce the block meta file size RandomAccessFile raf = new RandomAccessFile(metafile, "rw"); raf.SetLength(metafile.Length() - 20); raf.Close(); // restart DN to make replica to RWR MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0); cluster.RestartDataNode(dnProp, true); // try to recover the lease DistributedFileSystem newdfs = (DistributedFileSystem)FileSystem.NewInstance(cluster .GetConfiguration(0)); count = 0; while (++count < 10 && !newdfs.RecoverLease(file)) { Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsTrue("File should be closed", newdfs.RecoverLease(file)); }