public virtual void TestDeletionWithZeroSizeBlock() { Path foo = new Path("/foo"); Path bar = new Path(foo, "bar"); DFSTestUtil.CreateFile(hdfs, bar, Blocksize, Replication, 0L); SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s0"); hdfs.Append(bar); INodeFile barNode = fsdir.GetINode4Write(bar.ToString()).AsFile(); BlockInfoContiguous[] blks = barNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes()); ExtendedBlock previous = new ExtendedBlock(fsn.GetBlockPoolId(), blks[0]); cluster.GetNameNodeRpc().AddBlock(bar.ToString(), hdfs.GetClient().GetClientName( ), previous, null, barNode.GetId(), null); SnapshotTestHelper.CreateSnapshot(hdfs, foo, "s1"); barNode = fsdir.GetINode4Write(bar.ToString()).AsFile(); blks = barNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(2, blks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes()); NUnit.Framework.Assert.AreEqual(0, blks[1].GetNumBytes()); hdfs.Delete(bar, true); Path sbar = SnapshotTestHelper.GetSnapshotPath(foo, "s1", bar.GetName()); barNode = fsdir.GetINode(sbar.ToString()).AsFile(); blks = barNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(1, blks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, blks[0].GetNumBytes()); }
/// <summary>Append a file without closing the output stream</summary> /// <exception cref="System.IO.IOException"/> private HdfsDataOutputStream AppendFileWithoutClosing(Path file, int length) { byte[] toAppend = new byte[length]; Random random = new Random(); random.NextBytes(toAppend); HdfsDataOutputStream @out = (HdfsDataOutputStream)hdfs.Append(file); @out.Write(toAppend); return(@out); }
public virtual void TestFailedAppendBlockRejection() { Configuration conf = new HdfsConfiguration(); conf.Set("dfs.client.block.write.replace-datanode-on-failure.enable", "false"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); DistributedFileSystem fs = null; try { fs = cluster.GetFileSystem(); Path path = new Path("/test"); FSDataOutputStream @out = fs.Create(path); @out.WriteBytes("hello\n"); @out.Close(); // stop one datanode MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0); string dnAddress = dnProp.datanode.GetXferAddress().ToString(); if (dnAddress.StartsWith("/")) { dnAddress = Sharpen.Runtime.Substring(dnAddress, 1); } // append again to bump genstamps for (int i = 0; i < 2; i++) { @out = fs.Append(path); @out.WriteBytes("helloagain\n"); @out.Close(); } // re-open and make the block state as underconstruction @out = fs.Append(path); cluster.RestartDataNode(dnProp, true); // wait till the block report comes Sharpen.Thread.Sleep(2000); // check the block locations, this should not contain restarted datanode BlockLocation[] locations = fs.GetFileBlockLocations(path, 0, long.MaxValue); string[] names = locations[0].GetNames(); foreach (string node in names) { if (node.Equals(dnAddress)) { NUnit.Framework.Assert.Fail("Failed append should not be present in latest block locations." ); } } @out.Close(); } finally { IOUtils.CloseStream(fs); cluster.Shutdown(); } }
public virtual void TestAppend() { Configuration conf = new HdfsConfiguration(); short Replication = (short)3; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); try { DistributedFileSystem fs = cluster.GetFileSystem(); Path f = new Path(Dir, "testAppend"); { Log.Info("create an empty file " + f); fs.Create(f, Replication).Close(); FileStatus status = fs.GetFileStatus(f); NUnit.Framework.Assert.AreEqual(Replication, status.GetReplication()); NUnit.Framework.Assert.AreEqual(0L, status.GetLen()); } byte[] bytes = new byte[1000]; { Log.Info("append " + bytes.Length + " bytes to " + f); FSDataOutputStream @out = fs.Append(f); @out.Write(bytes); @out.Close(); FileStatus status = fs.GetFileStatus(f); NUnit.Framework.Assert.AreEqual(Replication, status.GetReplication()); NUnit.Framework.Assert.AreEqual(bytes.Length, status.GetLen()); } { Log.Info("append another " + bytes.Length + " bytes to " + f); try { FSDataOutputStream @out = fs.Append(f); @out.Write(bytes); @out.Close(); NUnit.Framework.Assert.Fail(); } catch (IOException ioe) { Log.Info("This exception is expected", ioe); } } } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Test the updation of NeededReplications for the Appended Block</summary> /// <exception cref="System.Exception"/> public virtual void TestUpdateNeededReplicationsForAppendedFile() { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); DistributedFileSystem fileSystem = null; try { // create a file. fileSystem = cluster.GetFileSystem(); Path f = new Path("/testAppend"); FSDataOutputStream create = fileSystem.Create(f, (short)2); create.Write(Sharpen.Runtime.GetBytesForString("/testAppend")); create.Close(); // Append to the file. FSDataOutputStream append = fileSystem.Append(f); append.Write(Sharpen.Runtime.GetBytesForString("/testAppend")); append.Close(); // Start a new datanode cluster.StartDataNodes(conf, 1, true, null, null); // Check for replications DFSTestUtil.WaitReplication(fileSystem, f, (short)2); } finally { if (null != fileSystem) { fileSystem.Close(); } cluster.Shutdown(); } }
/// <summary> /// The idea for making sure that there is no more than one instance /// running in an HDFS is to create a file in the HDFS, writes the hostname /// of the machine on which the instance is running to the file, but did not /// close the file until it exits. /// </summary> /// <remarks> /// The idea for making sure that there is no more than one instance /// running in an HDFS is to create a file in the HDFS, writes the hostname /// of the machine on which the instance is running to the file, but did not /// close the file until it exits. /// This prevents the second instance from running because it can not /// creates the file while the first one is running. /// This method checks if there is any running instance. If no, mark yes. /// Note that this is an atomic operation. /// </remarks> /// <returns> /// null if there is a running instance; /// otherwise, the output stream to the newly created file. /// </returns> /// <exception cref="System.IO.IOException"/> private OutputStream CheckAndMarkRunning() { try { if (fs.Exists(idPath)) { // try appending to it so that it will fail fast if another balancer is // running. IOUtils.CloseStream(fs.Append(idPath)); fs.Delete(idPath, true); } FSDataOutputStream fsout = fs.Create(idPath, false); // mark balancer idPath to be deleted during filesystem closure fs.DeleteOnExit(idPath); if (write2IdFile) { fsout.WriteBytes(Sharpen.Runtime.GetLocalHost().GetHostName()); fsout.Hflush(); } return(fsout); } catch (RemoteException e) { if (typeof(AlreadyBeingCreatedException).FullName.Equals(e.GetClassName())) { return(null); } else { throw; } } }
public virtual void TestAppend() { int maxOldFileLen = 2 * BlockSize + 1; int maxFlushedBytes = BlockSize; byte[] contents = AppendTestUtil.InitBuffer(maxOldFileLen + 2 * maxFlushedBytes); for (int oldFileLen = 0; oldFileLen <= maxOldFileLen; oldFileLen++) { for (int flushedBytes1 = 0; flushedBytes1 <= maxFlushedBytes; flushedBytes1++) { for (int flushedBytes2 = 0; flushedBytes2 <= maxFlushedBytes; flushedBytes2++) { int fileLen = oldFileLen + flushedBytes1 + flushedBytes2; // create the initial file of oldFileLen Path p = new Path("foo" + oldFileLen + "_" + flushedBytes1 + "_" + flushedBytes2); Log.Info("Creating file " + p); FSDataOutputStream @out = fs.Create(p, false, conf.GetInt(CommonConfigurationKeys .IoFileBufferSizeKey, 4096), Replication, BlockSize); @out.Write(contents, 0, oldFileLen); @out.Close(); // append flushedBytes bytes to the file @out = fs.Append(p); @out.Write(contents, oldFileLen, flushedBytes1); @out.Hflush(); // write another flushedBytes2 bytes to the file @out.Write(contents, oldFileLen + flushedBytes1, flushedBytes2); @out.Close(); // validate the file content AppendTestUtil.CheckFullFile(fs, p, fileLen, contents, p.ToString()); fs.Delete(p, false); } } } }
public virtual void TestAppendLessThanChecksumChunk() { byte[] buf = new byte[1024]; MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NumDataNodes (1).Build(); cluster.WaitActive(); try { using (DistributedFileSystem fs = cluster.GetFileSystem()) { int len1 = 200; int len2 = 300; Path p = new Path("/foo"); FSDataOutputStream @out = fs.Create(p); @out.Write(buf, 0, len1); @out.Close(); @out = fs.Append(p); @out.Write(buf, 0, len2); // flush but leave open @out.Hflush(); // read data to verify the replica's content and checksum are correct FSDataInputStream @in = fs.Open(p); int length = @in.Read(0, buf, 0, len1 + len2); NUnit.Framework.Assert.IsTrue(length > 0); @in.Close(); @out.Close(); } } finally { cluster.Shutdown(); } }
/// <exception cref="System.Exception"/> public virtual void TestFavoredNodesEndToEndForAppend() { // create 10 files with random preferred nodes for (int i = 0; i < NumFiles; i++) { Random rand = new Random(Runtime.CurrentTimeMillis() + i); // pass a new created rand so as to get a uniform distribution each time // without too much collisions (look at the do-while loop in getDatanodes) IPEndPoint[] datanode = GetDatanodes(rand); Path p = new Path("/filename" + i); // create and close the file. dfs.Create(p, FsPermission.GetDefault(), true, 4096, (short)3, 4096L, null, null) .Close(); // re-open for append FSDataOutputStream @out = dfs.Append(p, EnumSet.Of(CreateFlag.Append), 4096, null , datanode); @out.Write(SomeBytes); @out.Close(); BlockLocation[] locations = GetBlockLocations(p); // verify the files got created in the right nodes foreach (BlockLocation loc in locations) { string[] hosts = loc.GetNames(); string[] hosts1 = GetStringForInetSocketAddrs(datanode); NUnit.Framework.Assert.IsTrue(CompareNodes(hosts, hosts1)); } } }
public virtual void TestAppendWithPipelineRecovery() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; FSDataOutputStream @out = null; try { cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(true).ManageNameDfsDirs (true).NumDataNodes(4).Racks(new string[] { "/rack1", "/rack1", "/rack2", "/rack2" }).Build(); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); Path path = new Path("/test1"); @out = fs.Create(path, true, BlockSize, (short)3, BlockSize); AppendTestUtil.Write(@out, 0, 1024); @out.Close(); cluster.StopDataNode(3); @out = fs.Append(path); AppendTestUtil.Write(@out, 1024, 1024); @out.Close(); cluster.RestartNameNode(true); AppendTestUtil.Check(fs, path, 2048); } finally { IOUtils.CloseStream(@out); if (null != cluster) { cluster.Shutdown(); } } }
public virtual void Pipeline_01() { string MethodName = GenericTestUtils.GetMethodName(); if (Log.IsDebugEnabled()) { Log.Debug("Running " + MethodName); } Path filePath = new Path("/" + MethodName + ".dat"); DFSTestUtil.CreateFile(fs, filePath, FileSize, ReplFactor, rand.NextLong()); if (Log.IsDebugEnabled()) { Log.Debug("Invoking append but doing nothing otherwise..."); } FSDataOutputStream ofs = fs.Append(filePath); ofs.WriteBytes("Some more stuff to write"); ((DFSOutputStream)ofs.GetWrappedStream()).Hflush(); IList <LocatedBlock> lb = cluster.GetNameNodeRpc().GetBlockLocations(filePath.ToString (), FileSize - 1, FileSize).GetLocatedBlocks(); string bpid = cluster.GetNamesystem().GetBlockPoolId(); foreach (DataNode dn in cluster.GetDataNodes()) { Replica r = DataNodeTestUtils.FetchReplicaInfo(dn, bpid, lb[0].GetBlock().GetBlockId ()); NUnit.Framework.Assert.IsTrue("Replica on DN " + dn + " shouldn't be null", r != null); NUnit.Framework.Assert.AreEqual("Should be RBW replica on " + dn + " after sequence of calls append()/write()/hflush()" , HdfsServerConstants.ReplicaState.Rbw, r.GetState()); } ofs.Close(); }
public virtual void TestLeaseRecoveryAndAppend() { Configuration conf = new Configuration(); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); Path file = new Path("/testLeaseRecovery"); DistributedFileSystem dfs = cluster.GetFileSystem(); // create a file with 0 bytes FSDataOutputStream @out = dfs.Create(file); @out.Hflush(); @out.Hsync(); // abort the original stream ((DFSOutputStream)@out.GetWrappedStream()).Abort(); DistributedFileSystem newdfs = (DistributedFileSystem)FileSystem.NewInstance(cluster .GetConfiguration(0)); // Append to a file , whose lease is held by another client should fail try { newdfs.Append(file); NUnit.Framework.Assert.Fail("Append to a file(lease is held by another client) should fail" ); } catch (RemoteException e) { NUnit.Framework.Assert.IsTrue(e.Message.Contains("file lease is currently owned") ); } // Lease recovery on first try should be successful bool recoverLease = newdfs.RecoverLease(file); NUnit.Framework.Assert.IsTrue(recoverLease); FSDataOutputStream append = newdfs.Append(file); append.Write(Sharpen.Runtime.GetBytesForString("test")); append.Close(); } finally { if (cluster != null) { cluster.Shutdown(); cluster = null; } } }
/// <exception cref="System.IO.IOException"/> private void TestHSyncOperation(bool testWithAppend) { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); DistributedFileSystem fs = cluster.GetFileSystem(); Path p = new Path("/testHSync/foo"); int len = 1 << 16; FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag .Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null); if (testWithAppend) { // re-open the file with append call @out.Close(); @out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.SyncBlock), 4096, null ); } @out.Hflush(); // hflush does not sync CheckSyncMetric(cluster, 0); @out.Hsync(); // hsync on empty file does nothing CheckSyncMetric(cluster, 0); @out.Write(1); CheckSyncMetric(cluster, 0); @out.Hsync(); CheckSyncMetric(cluster, 1); // avoiding repeated hsyncs is a potential future optimization @out.Hsync(); CheckSyncMetric(cluster, 2); @out.Hflush(); // hflush still does not sync CheckSyncMetric(cluster, 2); @out.Close(); // close is sync'ing CheckSyncMetric(cluster, 3); // same with a file created with out SYNC_BLOCK @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag.Create, CreateFlag .Overwrite), 4096, (short)1, len, null); @out.Hsync(); CheckSyncMetric(cluster, 3); @out.Write(1); CheckSyncMetric(cluster, 3); @out.Hsync(); CheckSyncMetric(cluster, 4); // repeated hsyncs @out.Hsync(); CheckSyncMetric(cluster, 5); @out.Close(); // close does not sync (not opened with SYNC_BLOCK) CheckSyncMetric(cluster, 5); cluster.Shutdown(); }
/// <exception cref="System.Exception"/> public virtual void TestBlockReaderLocalLegacyWithAppend() { short ReplFactor = 1; HdfsConfiguration conf = GetConfiguration(null); conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); Path path = new Path("/testBlockReaderLocalLegacy"); DFSTestUtil.CreateFile(dfs, path, 10, ReplFactor, 0); DFSTestUtil.WaitReplication(dfs, path, ReplFactor); ClientDatanodeProtocol proxy; Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token; ExtendedBlock originalBlock; long originalGS; { LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString (), 0, 1).Get(0); proxy = DFSUtil.CreateClientDatanodeProtocolProxy(lb.GetLocations()[0], conf, 60000 , false); token = lb.GetBlockToken(); // get block and generation stamp ExtendedBlock blk = new ExtendedBlock(lb.GetBlock()); originalBlock = new ExtendedBlock(blk); originalGS = originalBlock.GetGenerationStamp(); // test getBlockLocalPathInfo BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(blk, token); NUnit.Framework.Assert.AreEqual(originalGS, info.GetBlock().GetGenerationStamp()); } { // append one byte FSDataOutputStream @out = dfs.Append(path); @out.Write(1); @out.Close(); } { // get new generation stamp LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString (), 0, 1).Get(0); long newGS = lb.GetBlock().GetGenerationStamp(); NUnit.Framework.Assert.IsTrue(newGS > originalGS); // getBlockLocalPathInfo using the original block. NUnit.Framework.Assert.AreEqual(originalGS, originalBlock.GetGenerationStamp()); BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(originalBlock, token); NUnit.Framework.Assert.AreEqual(newGS, info.GetBlock().GetGenerationStamp()); } cluster.Shutdown(); }
public virtual void TestAppend2AfterSoftLimit() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); //Set small soft-limit for lease long softLimit = 1L; long hardLimit = 9999999L; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.SetLeasePeriod(softLimit, hardLimit); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); DistributedFileSystem fs2 = new DistributedFileSystem(); fs2.Initialize(fs.GetUri(), conf); Path testPath = new Path("/testAppendAfterSoftLimit"); byte[] fileContents = AppendTestUtil.InitBuffer(32); // create a new file without closing FSDataOutputStream @out = fs.Create(testPath); @out.Write(fileContents); //Wait for > soft-limit Sharpen.Thread.Sleep(250); try { FSDataOutputStream appendStream2 = fs2.Append(testPath, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null); appendStream2.Write(fileContents); appendStream2.Close(); NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen ()); // make sure we now have 1 block since the first writer was revoked LocatedBlocks blks = fs.GetClient().GetLocatedBlocks(testPath.ToString(), 0L); NUnit.Framework.Assert.AreEqual(1, blks.GetLocatedBlocks().Count); foreach (LocatedBlock blk in blks.GetLocatedBlocks()) { NUnit.Framework.Assert.AreEqual(fileContents.Length, blk.GetBlockSize()); } } finally { fs.Close(); fs2.Close(); cluster.Shutdown(); } }
// Do small appends. /// <exception cref="System.IO.IOException"/> internal virtual void DoSmallAppends(Path file, DistributedFileSystem fs, int iterations ) { for (int i = 0; i < iterations; i++) { FSDataOutputStream stm; try { stm = fs.Append(file); } catch (IOException) { // If another thread is already appending, skip this time. continue; } // Failure in write or close will be terminal. AppendTestUtil.Write(stm, 0, 123); stm.Close(); } }
/// <exception cref="System.Exception"/> private void RecoverLeaseUsingCreate2(Path filepath) { FileSystem dfs2 = GetFSAsAnotherUser(conf); int size = AppendTestUtil.NextInt(FileSize); DistributedFileSystem dfsx = (DistributedFileSystem)dfs2; //create file using dfsx Path filepath2 = new Path("/immediateRecoverLease-x2"); FSDataOutputStream stm = dfsx.Create(filepath2, true, BufSize, ReplicationNum, BlockSize ); NUnit.Framework.Assert.IsTrue(dfsx.dfs.Exists("/immediateRecoverLease-x2")); try { Sharpen.Thread.Sleep(10000); } catch (Exception) { } dfsx.Append(filepath); }
public virtual void TestAppendAfterSoftLimit() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1); conf.SetBoolean(DFSConfigKeys.DfsSupportAppendKey, true); //Set small soft-limit for lease long softLimit = 1L; long hardLimit = 9999999L; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.SetLeasePeriod(softLimit, hardLimit); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); FileSystem fs2 = new DistributedFileSystem(); fs2.Initialize(fs.GetUri(), conf); Path testPath = new Path("/testAppendAfterSoftLimit"); byte[] fileContents = AppendTestUtil.InitBuffer(32); // create a new file without closing FSDataOutputStream @out = fs.Create(testPath); @out.Write(fileContents); //Wait for > soft-limit Sharpen.Thread.Sleep(250); try { FSDataOutputStream appendStream2 = fs2.Append(testPath); appendStream2.Write(fileContents); appendStream2.Close(); NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen ()); } finally { fs.Close(); fs2.Close(); cluster.Shutdown(); } }
public virtual void TestAppend2Twice() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); DistributedFileSystem fs1 = cluster.GetFileSystem(); FileSystem fs2 = AppendTestUtil.CreateHdfsWithDifferentUsername(conf); try { Path p = new Path("/testAppendTwice/foo"); int len = 1 << 16; byte[] fileContents = AppendTestUtil.InitBuffer(len); { // create a new file with a full block. FSDataOutputStream @out = fs2.Create(p, true, 4096, (short)1, len); @out.Write(fileContents, 0, len); @out.Close(); } //1st append does not add any data so that the last block remains full //and the last block in INodeFileUnderConstruction is a BlockInfo //but not BlockInfoUnderConstruction. ((DistributedFileSystem)fs2).Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock ), 4096, null); // 2nd append should get AlreadyBeingCreatedException fs1.Append(p); NUnit.Framework.Assert.Fail(); } catch (RemoteException re) { AppendTestUtil.Log.Info("Got an exception:", re); NUnit.Framework.Assert.AreEqual(typeof(AlreadyBeingCreatedException).FullName, re .GetClassName()); } finally { fs2.Close(); fs1.Close(); cluster.Shutdown(); } }
/// <summary> /// Test if the quota can be correctly updated when file length is updated /// through fsync /// </summary> /// <exception cref="System.Exception"/> public virtual void TestUpdateQuotaForFSync() { Path foo = new Path("/foo"); Path bar = new Path(foo, "bar"); DFSTestUtil.CreateFile(dfs, bar, Blocksize, Replication, 0L); dfs.SetQuota(foo, long.MaxValue - 1, long.MaxValue - 1); FSDataOutputStream @out = dfs.Append(bar); @out.Write(new byte[Blocksize / 4]); ((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .UpdateLength)); INodeDirectory fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory(); QuotaCounts quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed(); long ns = quota.GetNameSpace(); long ds = quota.GetStorageSpace(); NUnit.Framework.Assert.AreEqual(2, ns); // foo and bar NUnit.Framework.Assert.AreEqual(Blocksize * 2 * Replication, ds); // file is under construction @out.Write(new byte[Blocksize / 4]); @out.Close(); fooNode = fsdir.GetINode4Write(foo.ToString()).AsDirectory(); quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed(); ns = quota.GetNameSpace(); ds = quota.GetStorageSpace(); NUnit.Framework.Assert.AreEqual(2, ns); NUnit.Framework.Assert.AreEqual((Blocksize + Blocksize / 2) * Replication, ds); // append another block DFSTestUtil.AppendFile(dfs, bar, Blocksize); quota = fooNode.GetDirectoryWithQuotaFeature().GetSpaceConsumed(); ns = quota.GetNameSpace(); ds = quota.GetStorageSpace(); NUnit.Framework.Assert.AreEqual(2, ns); // foo and bar NUnit.Framework.Assert.AreEqual((Blocksize * 2 + Blocksize / 2) * Replication, ds ); }
public virtual void TestAddBlockUC() { DistributedFileSystem fs = cluster.GetFileSystem(); Path file1 = new Path("/file1"); DFSTestUtil.CreateFile(fs, file1, Blocksize - 1, Replication, 0L); FSDataOutputStream @out = null; try { // append files without closing the streams @out = fs.Append(file1); string appendContent = "appending-content"; @out.WriteBytes(appendContent); ((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag .UpdateLength)); // restart NN cluster.RestartNameNode(true); FSDirectory fsdir = cluster.GetNamesystem().GetFSDirectory(); INodeFile fileNode = fsdir.GetINode4Write(file1.ToString()).AsFile(); BlockInfoContiguous[] fileBlocks = fileNode.GetBlocks(); NUnit.Framework.Assert.AreEqual(2, fileBlocks.Length); NUnit.Framework.Assert.AreEqual(Blocksize, fileBlocks[0].GetNumBytes()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.Complete, fileBlocks [0].GetBlockUCState()); NUnit.Framework.Assert.AreEqual(appendContent.Length - 1, fileBlocks[1].GetNumBytes ()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.UnderConstruction , fileBlocks[1].GetBlockUCState()); } finally { if (@out != null) { @out.Close(); } } }
public virtual void TestBestEffort() { Configuration conf = new HdfsConfiguration(); //always replace a datanode but do not throw exception ReplaceDatanodeOnFailure.Write(ReplaceDatanodeOnFailure.Policy.Always, true, conf ); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); try { DistributedFileSystem fs = cluster.GetFileSystem(); Path f = new Path(Dir, "testIgnoreReplaceFailure"); byte[] bytes = new byte[1000]; { Log.Info("write " + bytes.Length + " bytes to " + f); FSDataOutputStream @out = fs.Create(f, Replication); @out.Write(bytes); @out.Close(); FileStatus status = fs.GetFileStatus(f); NUnit.Framework.Assert.AreEqual(Replication, status.GetReplication()); NUnit.Framework.Assert.AreEqual(bytes.Length, status.GetLen()); } { Log.Info("append another " + bytes.Length + " bytes to " + f); FSDataOutputStream @out = fs.Append(f); @out.Write(bytes); @out.Close(); } } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestSpaceCommands() { Configuration conf = new HdfsConfiguration(); // set a smaller block size so that we can test with smaller // diskspace quotas conf.Set(DFSConfigKeys.DfsBlockSizeKey, "512"); // Make it relinquish locks. When run serially, the result should // be identical. conf.SetInt(DFSConfigKeys.DfsContentSummaryLimitKey, 2); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue("Not a HDFS: " + fs.GetUri(), fs is DistributedFileSystem ); DistributedFileSystem dfs = (DistributedFileSystem)fs; try { int fileLen = 1024; short replication = 3; int fileSpace = fileLen * replication; // create directory /nqdir0/qdir1/qdir20/nqdir30 NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(new Path("/nqdir0/qdir1/qdir20/nqdir30") )); // set the quota of /nqdir0/qdir1 to 4 * fileSpace Path quotaDir1 = new Path("/nqdir0/qdir1"); dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 4 * fileSpace); ContentSummary c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 4 * fileSpace); // set the quota of /nqdir0/qdir1/qdir20 to 6 * fileSpace Path quotaDir20 = new Path("/nqdir0/qdir1/qdir20"); dfs.SetQuota(quotaDir20, HdfsConstants.QuotaDontSet, 6 * fileSpace); c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 6 * fileSpace); // Create /nqdir0/qdir1/qdir21 and set its space quota to 2 * fileSpace Path quotaDir21 = new Path("/nqdir0/qdir1/qdir21"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir21)); dfs.SetQuota(quotaDir21, HdfsConstants.QuotaDontSet, 2 * fileSpace); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 2 * fileSpace); // 5: Create directory /nqdir0/qdir1/qdir21/nqdir32 Path tempPath = new Path(quotaDir21, "nqdir32"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(tempPath)); // create a file under nqdir32/fileDir DFSTestUtil.CreateFile(dfs, new Path(tempPath, "fileDir/file1"), fileLen, replication , 0); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace); // Create a larger file /nqdir0/qdir1/qdir21/nqdir33/ bool hasException = false; try { DFSTestUtil.CreateFile(dfs, new Path(quotaDir21, "nqdir33/file2"), 2 * fileLen, replication , 0); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // delete nqdir33 NUnit.Framework.Assert.IsTrue(dfs.Delete(new Path(quotaDir21, "nqdir33"), true)); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 2 * fileSpace); // Verify space before the move: c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0); // Move /nqdir0/qdir1/qdir21/nqdir32 /nqdir0/qdir1/qdir20/nqdir30 Path dstPath = new Path(quotaDir20, "nqdir30"); Path srcPath = new Path(quotaDir21, "nqdir32"); NUnit.Framework.Assert.IsTrue(dfs.Rename(srcPath, dstPath)); // verify space after the move c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace); // verify space for its parent c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), fileSpace); // verify space for source for the move c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0); Path file2 = new Path(dstPath, "fileDir/file2"); int file2Len = 2 * fileLen; // create a larger file under /nqdir0/qdir1/qdir20/nqdir30 DFSTestUtil.CreateFile(dfs, file2, file2Len, replication, 0); c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0); // Reverse: Move /nqdir0/qdir1/qdir20/nqdir30 to /nqdir0/qdir1/qdir21/ hasException = false; try { NUnit.Framework.Assert.IsFalse(dfs.Rename(dstPath, srcPath)); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // make sure no intermediate directories left by failed rename NUnit.Framework.Assert.IsFalse(dfs.Exists(srcPath)); // directory should exist NUnit.Framework.Assert.IsTrue(dfs.Exists(dstPath)); // verify space after the failed move c = dfs.GetContentSummary(quotaDir20); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace); c = dfs.GetContentSummary(quotaDir21); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 0); // Test Append : // verify space quota c = dfs.GetContentSummary(quotaDir1); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), 4 * fileSpace); // verify space before append; c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 3 * fileSpace); OutputStream @out = dfs.Append(file2); // appending 1 fileLen should succeed @out.Write(new byte[fileLen]); @out.Close(); file2Len += fileLen; // after append // verify space after append; c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 4 * fileSpace); // now increase the quota for quotaDir1 dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 5 * fileSpace); // Now, appending more than 1 fileLen should result in an error @out = dfs.Append(file2); hasException = false; try { @out.Write(new byte[fileLen + 1024]); @out.Flush(); @out.Close(); } catch (DSQuotaExceededException) { hasException = true; IOUtils.CloseStream(@out); } NUnit.Framework.Assert.IsTrue(hasException); file2Len += fileLen; // after partial append // verify space after partial append c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace); // Test set replication : // first reduce the replication dfs.SetReplication(file2, (short)(replication - 1)); // verify that space is reduced by file2Len c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace - file2Len); // now try to increase the replication and and expect an error. hasException = false; try { dfs.SetReplication(file2, (short)(replication + 1)); } catch (DSQuotaExceededException) { hasException = true; } NUnit.Framework.Assert.IsTrue(hasException); // verify space consumed remains unchanged. c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace - file2Len); // now increase the quota for quotaDir1 and quotaDir20 dfs.SetQuota(quotaDir1, HdfsConstants.QuotaDontSet, 10 * fileSpace); dfs.SetQuota(quotaDir20, HdfsConstants.QuotaDontSet, 10 * fileSpace); // then increasing replication should be ok. dfs.SetReplication(file2, (short)(replication + 1)); // verify increase in space c = dfs.GetContentSummary(dstPath); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), 5 * fileSpace + file2Len); // Test HDFS-2053 : // Create directory /hdfs-2053 Path quotaDir2053 = new Path("/hdfs-2053"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053)); // Create subdirectories /hdfs-2053/{A,B,C} Path quotaDir2053_A = new Path(quotaDir2053, "A"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_A)); Path quotaDir2053_B = new Path(quotaDir2053, "B"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_B)); Path quotaDir2053_C = new Path(quotaDir2053, "C"); NUnit.Framework.Assert.IsTrue(dfs.Mkdirs(quotaDir2053_C)); // Factors to vary the sizes of test files created in each subdir. // The actual factors are not really important but they allow us to create // identifiable file sizes per subdir, which helps during debugging. int sizeFactorA = 1; int sizeFactorB = 2; int sizeFactorC = 4; // Set space quota for subdirectory C dfs.SetQuota(quotaDir2053_C, HdfsConstants.QuotaDontSet, (sizeFactorC + 1) * fileSpace ); c = dfs.GetContentSummary(quotaDir2053_C); NUnit.Framework.Assert.AreEqual(c.GetSpaceQuota(), (sizeFactorC + 1) * fileSpace); // Create a file under subdirectory A DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_A, "fileA"), sizeFactorA * fileLen , replication, 0); c = dfs.GetContentSummary(quotaDir2053_A); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorA * fileSpace); // Create a file under subdirectory B DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_B, "fileB"), sizeFactorB * fileLen , replication, 0); c = dfs.GetContentSummary(quotaDir2053_B); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorB * fileSpace); // Create a file under subdirectory C (which has a space quota) DFSTestUtil.CreateFile(dfs, new Path(quotaDir2053_C, "fileC"), sizeFactorC * fileLen , replication, 0); c = dfs.GetContentSummary(quotaDir2053_C); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), sizeFactorC * fileSpace); // Check space consumed for /hdfs-2053 c = dfs.GetContentSummary(quotaDir2053); NUnit.Framework.Assert.AreEqual(c.GetSpaceConsumed(), (sizeFactorA + sizeFactorB + sizeFactorC) * fileSpace); NUnit.Framework.Assert.AreEqual(20, cluster.GetNamesystem().GetFSDirectory().GetYieldCount ()); } finally { cluster.Shutdown(); } }
public virtual void TestMultiAppend2() { Configuration conf = new HdfsConfiguration(); conf.Set("dfs.client.block.write.replace-datanode-on-failure.enable", "false"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); DistributedFileSystem fs = null; string hello = "hello\n"; try { fs = cluster.GetFileSystem(); Path path = new Path("/test"); FSDataOutputStream @out = fs.Create(path); @out.WriteBytes(hello); @out.Close(); // stop one datanode MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0); string dnAddress = dnProp.datanode.GetXferAddress().ToString(); if (dnAddress.StartsWith("/")) { dnAddress = Sharpen.Runtime.Substring(dnAddress, 1); } // append again to bump genstamps for (int i = 0; i < 2; i++) { @out = fs.Append(path, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096, null); @out.WriteBytes(hello); @out.Close(); } // re-open and make the block state as underconstruction @out = fs.Append(path, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096, null); cluster.RestartDataNode(dnProp, true); // wait till the block report comes Sharpen.Thread.Sleep(2000); @out.WriteBytes(hello); @out.Close(); // check the block locations LocatedBlocks blocks = fs.GetClient().GetLocatedBlocks(path.ToString(), 0L); // since we append the file 3 time, we should be 4 blocks NUnit.Framework.Assert.AreEqual(4, blocks.GetLocatedBlocks().Count); foreach (LocatedBlock block in blocks.GetLocatedBlocks()) { NUnit.Framework.Assert.AreEqual(hello.Length, block.GetBlockSize()); } StringBuilder sb = new StringBuilder(); for (int i_1 = 0; i_1 < 4; i_1++) { sb.Append(hello); } byte[] content = Sharpen.Runtime.GetBytesForString(sb.ToString()); AppendTestUtil.CheckFullFile(fs, path, content.Length, content, "Read /test"); // restart namenode to make sure the editlog can be properly applied cluster.RestartNameNode(true); cluster.WaitActive(); AppendTestUtil.CheckFullFile(fs, path, content.Length, content, "Read /test"); blocks = fs.GetClient().GetLocatedBlocks(path.ToString(), 0L); // since we append the file 3 time, we should be 4 blocks NUnit.Framework.Assert.AreEqual(4, blocks.GetLocatedBlocks().Count); foreach (LocatedBlock block_1 in blocks.GetLocatedBlocks()) { NUnit.Framework.Assert.AreEqual(hello.Length, block_1.GetBlockSize()); } } finally { IOUtils.CloseStream(fs); cluster.Shutdown(); } }
public virtual void TestTC1() { Path p = new Path("/TC1/foo"); System.Console.Out.WriteLine("p=" + p); //a. Create file and write one block of data. Close file. int len1 = (int)BlockSize; { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } // Reopen file to append. Append half block of data. Close file. int len2 = (int)BlockSize / 2; { FSDataOutputStream @out = fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //b. Reopen file and read 1.5 blocks worth of data. Close file. AppendTestUtil.Check(fs, p, len1 + len2); }
// create a bunch of files. Write to them and then verify. public override void Run() { System.Console.Out.WriteLine("Workload " + this.id + " starting... "); for (int i = 0; i < this._enclosing.numAppendsPerThread; i++) { // pick a file at random and remove it from pool Path testfile; lock (this._enclosing.testFiles) { if (this._enclosing.testFiles.Count == 0) { System.Console.Out.WriteLine("Completed write to almost all files."); return; } int index = AppendTestUtil.NextInt(this._enclosing.testFiles.Count); testfile = this._enclosing.testFiles.Remove(index); } long len = 0; int sizeToAppend = 0; try { DistributedFileSystem fs = this.cluster.GetFileSystem(); // add a random number of bytes to file len = fs.GetFileStatus(testfile).GetLen(); // if file is already full, then pick another file if (len >= AppendTestUtil.FileSize) { System.Console.Out.WriteLine("File " + testfile + " is full."); continue; } // do small size appends so that we can trigger multiple // appends to the same file. // int left = (int)(AppendTestUtil.FileSize - len) / 3; if (left <= 0) { left = 1; } sizeToAppend = AppendTestUtil.NextInt(left); System.Console.Out.WriteLine("Workload thread " + this.id + " appending " + sizeToAppend + " bytes " + " to file " + testfile + " of size " + len); FSDataOutputStream stm = this.appendToNewBlock ? fs.Append(testfile, EnumSet.Of(CreateFlag .Append, CreateFlag.NewBlock), 4096, null) : fs.Append(testfile); stm.Write(this._enclosing.fileContents, (int)len, sizeToAppend); stm.Close(); // wait for the file size to be reflected in the namenode metadata while (fs.GetFileStatus(testfile).GetLen() != (len + sizeToAppend)) { try { System.Console.Out.WriteLine("Workload thread " + this.id + " file " + testfile + " size " + fs.GetFileStatus(testfile).GetLen() + " expected size " + (len + sizeToAppend ) + " waiting for namenode metadata update."); Sharpen.Thread.Sleep(5000); } catch (Exception) { } } NUnit.Framework.Assert.IsTrue("File " + testfile + " size is " + fs.GetFileStatus (testfile).GetLen() + " but expected " + (len + sizeToAppend), fs.GetFileStatus( testfile).GetLen() == (len + sizeToAppend)); AppendTestUtil.CheckFullFile(fs, testfile, (int)(len + sizeToAppend), this._enclosing .fileContents, "Read 2"); } catch (Exception e) { TestFileAppend2.globalStatus = false; if (e.ToString() != null) { System.Console.Out.WriteLine("Workload exception " + this.id + " testfile " + testfile + " " + e); Sharpen.Runtime.PrintStackTrace(e); } NUnit.Framework.Assert.IsTrue("Workload exception " + this.id + " testfile " + testfile + " expected size " + (len + sizeToAppend), false); } // Add testfile back to the pool of files. lock (this._enclosing.testFiles) { this._enclosing.testFiles.AddItem(testfile); } } }
public virtual void TestSimpleAppend2() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsDatanodeHandlerCountKey, 50); fileContents = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); DistributedFileSystem fs = cluster.GetFileSystem(); try { { // test appending to a file. // create a new file. Path file1 = new Path("/simpleAppend.dat"); FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1); System.Console.Out.WriteLine("Created file simpleAppend.dat"); // write to file int mid = 186; // io.bytes.per.checksum bytes System.Console.Out.WriteLine("Writing " + mid + " bytes to file " + file1); stm.Write(fileContents, 0, mid); stm.Close(); System.Console.Out.WriteLine("Wrote and Closed first part of file."); // write to file int mid2 = 607; // io.bytes.per.checksum bytes System.Console.Out.WriteLine("Writing " + mid + " bytes to file " + file1); stm = fs.Append(file1, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096, null); stm.Write(fileContents, mid, mid2 - mid); stm.Close(); System.Console.Out.WriteLine("Wrote and Closed second part of file."); // write the remainder of the file stm = fs.Append(file1, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096, null); // ensure getPos is set to reflect existing size of the file NUnit.Framework.Assert.IsTrue(stm.GetPos() > 0); System.Console.Out.WriteLine("Writing " + (AppendTestUtil.FileSize - mid2) + " bytes to file " + file1); stm.Write(fileContents, mid2, AppendTestUtil.FileSize - mid2); System.Console.Out.WriteLine("Written second part of file"); stm.Close(); System.Console.Out.WriteLine("Wrote and Closed second part of file."); // verify that entire file is good AppendTestUtil.CheckFullFile(fs, file1, AppendTestUtil.FileSize, fileContents, "Read 2" ); // also make sure there three different blocks for the file IList <LocatedBlock> blocks = fs.GetClient().GetLocatedBlocks(file1.ToString(), 0L ).GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(12, blocks.Count); // the block size is 1024 NUnit.Framework.Assert.AreEqual(mid, blocks[0].GetBlockSize()); NUnit.Framework.Assert.AreEqual(mid2 - mid, blocks[1].GetBlockSize()); for (int i = 2; i < 11; i++) { NUnit.Framework.Assert.AreEqual(AppendTestUtil.BlockSize, blocks[i].GetBlockSize( )); } NUnit.Framework.Assert.AreEqual((AppendTestUtil.FileSize - mid2) % AppendTestUtil .BlockSize, blocks[11].GetBlockSize()); } { // test appending to an non-existing file. FSDataOutputStream @out = null; try { @out = fs.Append(new Path("/non-existing.dat"), EnumSet.Of(CreateFlag.Append, CreateFlag .NewBlock), 4096, null); NUnit.Framework.Assert.Fail("Expected to have FileNotFoundException"); } catch (FileNotFoundException fnfe) { System.Console.Out.WriteLine("Good: got " + fnfe); Sharpen.Runtime.PrintStackTrace(fnfe, System.Console.Out); } finally { IOUtils.CloseStream(@out); } } { // test append permission. // set root to all writable Path root = new Path("/"); fs.SetPermission(root, new FsPermission((short)0x1ff)); fs.Close(); // login as a different user UserGroupInformation superuser = UserGroupInformation.GetCurrentUser(); string username = "******"; string group = "testappendgroup"; NUnit.Framework.Assert.IsFalse(superuser.GetShortUserName().Equals(username)); NUnit.Framework.Assert.IsFalse(Arrays.AsList(superuser.GetGroupNames()).Contains( group)); UserGroupInformation appenduser = UserGroupInformation.CreateUserForTesting(username , new string[] { group }); fs = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(appenduser, conf); // create a file Path dir = new Path(root, GetType().Name); Path foo = new Path(dir, "foo.dat"); FSDataOutputStream @out = null; int offset = 0; try { @out = fs.Create(foo); int len = 10 + AppendTestUtil.NextInt(100); @out.Write(fileContents, offset, len); offset += len; } finally { IOUtils.CloseStream(@out); } // change dir and foo to minimal permissions. fs.SetPermission(dir, new FsPermission((short)0x40)); fs.SetPermission(foo, new FsPermission((short)0x80)); // try append, should success @out = null; try { @out = fs.Append(foo, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096, null ); int len = 10 + AppendTestUtil.NextInt(100); @out.Write(fileContents, offset, len); offset += len; } finally { IOUtils.CloseStream(@out); } // change dir and foo to all but no write on foo. fs.SetPermission(foo, new FsPermission((short)0x17f)); fs.SetPermission(dir, new FsPermission((short)0x1ff)); // try append, should fail @out = null; try { @out = fs.Append(foo, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096, null ); NUnit.Framework.Assert.Fail("Expected to have AccessControlException"); } catch (AccessControlException ace) { System.Console.Out.WriteLine("Good: got " + ace); Sharpen.Runtime.PrintStackTrace(ace, System.Console.Out); } finally { IOUtils.CloseStream(@out); } } } finally { fs.Close(); cluster.Shutdown(); } }
/// <summary> /// Test that we cannot read a file beyond its snapshot length /// when accessing it via a snapshot path. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotfileLength() { hdfs.Mkdirs(sub); int bytesRead; byte[] buffer = new byte[Blocksize * 8]; int origLen = Blocksize + 1; int toAppend = Blocksize; FSDataInputStream fis = null; FileStatus fileStatus = null; // Create and write a file. Path file1 = new Path(sub, file1Name); DFSTestUtil.CreateFile(hdfs, file1, Blocksize, 0, Blocksize, Replication, Seed); DFSTestUtil.AppendFile(hdfs, file1, origLen); // Create a snapshot on the parent directory. hdfs.AllowSnapshot(sub); hdfs.CreateSnapshot(sub, snapshot1); Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name); FileChecksum snapChksum1 = hdfs.GetFileChecksum(file1snap1); Assert.AssertThat("file and snapshot file checksums are not equal", hdfs.GetFileChecksum (file1), CoreMatchers.Is(snapChksum1)); // Append to the file. FSDataOutputStream @out = hdfs.Append(file1); // Nothing has been appended yet. All checksums should still be equal. Assert.AssertThat("file and snapshot checksums (open for append) are not equal", hdfs.GetFileChecksum(file1), CoreMatchers.Is(snapChksum1)); Assert.AssertThat("snapshot checksum (post-open for append) has changed", hdfs.GetFileChecksum (file1snap1), CoreMatchers.Is(snapChksum1)); try { AppendTestUtil.Write(@out, 0, toAppend); // Test reading from snapshot of file that is open for append byte[] dataFromSnapshot = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1); Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot.Length, CoreMatchers.Is (origLen)); // Verify that checksum didn't change Assert.AssertThat("snapshot file checksum (pre-close) has changed", hdfs.GetFileChecksum (file1), CoreMatchers.Is(snapChksum1)); Assert.AssertThat("snapshot checksum (post-append) has changed", hdfs.GetFileChecksum (file1snap1), CoreMatchers.Is(snapChksum1)); } finally { @out.Close(); } Assert.AssertThat("file and snapshot file checksums (post-close) are equal", hdfs .GetFileChecksum(file1), CoreMatchers.Not(snapChksum1)); Assert.AssertThat("snapshot file checksum (post-close) has changed", hdfs.GetFileChecksum (file1snap1), CoreMatchers.Is(snapChksum1)); // Make sure we can read the entire file via its non-snapshot path. fileStatus = hdfs.GetFileStatus(file1); Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen + toAppend)); fis = hdfs.Open(file1); bytesRead = fis.Read(0, buffer, 0, buffer.Length); Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen + toAppend)); fis.Close(); // Try to open the file via its snapshot path. fis = hdfs.Open(file1snap1); fileStatus = hdfs.GetFileStatus(file1snap1); Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen)); // Make sure we can only read up to the snapshot length. bytesRead = fis.Read(0, buffer, 0, buffer.Length); Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen)); fis.Close(); byte[] dataFromSnapshot_1 = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1); Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot_1.Length, CoreMatchers.Is (origLen)); }
/// <summary> /// Test that an append with no locations fails with an exception /// showing insufficient locations. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestAppendInsufficientLocations() { Configuration conf = new Configuration(); // lower heartbeat interval for fast recognition of DN conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1000); conf.SetInt(DFSConfigKeys.DfsHeartbeatIntervalKey, 1); conf.SetInt(DFSConfigKeys.DfsClientSocketTimeoutKey, 3000); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build(); DistributedFileSystem fileSystem = null; try { // create a file with replication 3 fileSystem = cluster.GetFileSystem(); Path f = new Path("/testAppend"); FSDataOutputStream create = fileSystem.Create(f, (short)2); create.Write(Sharpen.Runtime.GetBytesForString("/testAppend")); create.Close(); // Check for replications DFSTestUtil.WaitReplication(fileSystem, f, (short)2); // Shut down all DNs that have the last block location for the file LocatedBlocks lbs = fileSystem.dfs.GetNamenode().GetBlockLocations("/testAppend", 0, long.MaxValue); IList <DataNode> dnsOfCluster = cluster.GetDataNodes(); DatanodeInfo[] dnsWithLocations = lbs.GetLastLocatedBlock().GetLocations(); foreach (DataNode dn in dnsOfCluster) { foreach (DatanodeInfo loc in dnsWithLocations) { if (dn.GetDatanodeId().Equals(loc)) { dn.Shutdown(); DFSTestUtil.WaitForDatanodeDeath(dn); } } } // Wait till 0 replication is recognized DFSTestUtil.WaitReplication(fileSystem, f, (short)0); // Append to the file, at this state there are 3 live DNs but none of them // have the block. try { fileSystem.Append(f); NUnit.Framework.Assert.Fail("Append should fail because insufficient locations"); } catch (IOException e) { Log.Info("Expected exception: ", e); } FSDirectory dir = cluster.GetNamesystem().GetFSDirectory(); INodeFile inode = INodeFile.ValueOf(dir.GetINode("/testAppend"), "/testAppend"); NUnit.Framework.Assert.IsTrue("File should remain closed", !inode.IsUnderConstruction ()); } finally { if (null != fileSystem) { fileSystem.Close(); } cluster.Shutdown(); } }
/// <exception cref="System.Exception"/> public virtual void TestAppend() { fs.Append(objInSnapshot, 65535, null); }