/// <summary> /// Tests the fileLength when we sync the file and restart the cluster and /// Datanodes not report to Namenode yet. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestFileLengthWithHSyncAndClusterRestartWithOutDNsRegister() { Configuration conf = new HdfsConfiguration(); // create cluster conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, 512); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); HdfsDataInputStream @in = null; try { Path path = new Path("/tmp/TestFileLengthOnClusterRestart", "test"); DistributedFileSystem dfs = cluster.GetFileSystem(); FSDataOutputStream @out = dfs.Create(path); int fileLength = 1030; @out.Write(new byte[fileLength]); @out.Hsync(); cluster.RestartNameNode(); cluster.WaitActive(); @in = (HdfsDataInputStream)dfs.Open(path, 1024); // Verify the length when we just restart NN. DNs will register // immediately. NUnit.Framework.Assert.AreEqual(fileLength, @in.GetVisibleLength()); cluster.ShutdownDataNodes(); cluster.RestartNameNode(false); // This is just for ensuring NN started. VerifyNNIsInSafeMode(dfs); try { @in = (HdfsDataInputStream)dfs.Open(path); NUnit.Framework.Assert.Fail("Expected IOException"); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e.GetLocalizedMessage().IndexOf("Name node is in safe mode" ) >= 0); } } finally { if (null != @in) { @in.Close(); } cluster.Shutdown(); } }
//check the file /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal static void CheckFile(Path p, int expectedsize, Configuration conf) { //open the file with another user account string username = UserGroupInformation.GetCurrentUser().GetShortUserName() + "_" + ++userCount; UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(username, new string[] { "supergroup" }); FileSystem fs = DFSTestUtil.GetFileSystemAs(ugi, conf); HdfsDataInputStream @in = (HdfsDataInputStream)fs.Open(p); //Check visible length NUnit.Framework.Assert.IsTrue(@in.GetVisibleLength() >= expectedsize); //Able to read? for (int i = 0; i < expectedsize; i++) { NUnit.Framework.Assert.AreEqual(unchecked ((byte)i), unchecked ((byte)@in.Read())); } @in.Close(); }
/// <exception cref="System.IO.IOException"/> private void OnOpen(ChannelHandlerContext ctx) { string nnId = @params.NamenodeId(); int bufferSize = @params.BufferSize(); long offset = @params.Offset(); long length = @params.Length(); DefaultHttpResponse response = new DefaultHttpResponse(HttpVersion.Http11, HttpResponseStatus .Ok); HttpHeaders headers = response.Headers(); // Allow the UI to access the file headers.Set(HttpHeaders.Names.AccessControlAllowMethods, HttpMethod.Get); headers.Set(HttpHeaders.Names.AccessControlAllowOrigin, "*"); headers.Set(HttpHeaders.Names.ContentType, ApplicationOctetStream); headers.Set(HttpHeaders.Names.Connection, HttpHeaders.Values.Close); DFSClient dfsclient = NewDfsClient(nnId, conf); HdfsDataInputStream @in = dfsclient.CreateWrappedInputStream(dfsclient.Open(path, bufferSize, true)); @in.Seek(offset); long contentLength = @in.GetVisibleLength() - offset; if (length >= 0) { contentLength = Math.Min(contentLength, length); } InputStream data; if (contentLength >= 0) { headers.Set(HttpHeaders.Names.ContentLength, contentLength); data = new LimitInputStream(@in, contentLength); } else { data = @in; } ctx.Write(response); ctx.WriteAndFlush(new _ChunkedStream_221(dfsclient, data)).AddListener(ChannelFutureListener .Close); }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { string path = ServletUtil.GetDecodedPath(request, "/streamFile"); string rawPath = ServletUtil.GetRawPath(request, "/streamFile"); string filename = JspHelper.ValidatePath(path); string rawFilename = JspHelper.ValidatePath(rawPath); if (filename == null) { response.SetContentType("text/plain"); PrintWriter @out = response.GetWriter(); @out.Write("Invalid input"); return; } Enumeration <string> reqRanges = request.GetHeaders("Range"); if (reqRanges != null && !reqRanges.MoveNext()) { reqRanges = null; } DFSClient dfs; try { dfs = GetDFSClient(request); } catch (Exception e) { response.SendError(400, e.Message); return; } HdfsDataInputStream @in = null; OutputStream out_1 = null; try { @in = dfs.CreateWrappedInputStream(dfs.Open(filename)); out_1 = response.GetOutputStream(); long fileLen = @in.GetVisibleLength(); if (reqRanges != null) { IList <InclusiveByteRange> ranges = InclusiveByteRange.SatisfiableRanges(reqRanges , fileLen); StreamFile.SendPartialData(@in, out_1, response, fileLen, ranges); } else { // No ranges, so send entire file response.SetHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\""); response.SetContentType("application/octet-stream"); response.SetHeader(ContentLength, string.Empty + fileLen); StreamFile.CopyFromOffset(@in, out_1, 0L, fileLen); } @in.Close(); @in = null; out_1.Close(); out_1 = null; dfs.Close(); dfs = null; } catch (IOException ioe) { if (Log.IsDebugEnabled()) { Log.Debug("response.isCommitted()=" + response.IsCommitted(), ioe); } throw; } finally { IOUtils.Cleanup(Log, @in); IOUtils.Cleanup(Log, out_1); IOUtils.Cleanup(Log, dfs); } }
/// <exception cref="System.Exception"/> public virtual void HardLeaseRecoveryRestartHelper(bool doRename, int size) { if (size < 0) { size = AppendTestUtil.NextInt(FileSize + 1); } //create a file string fileStr = "/hardLeaseRecovery"; AppendTestUtil.Log.Info("filestr=" + fileStr); Path filePath = new Path(fileStr); FSDataOutputStream stm = dfs.Create(filePath, true, BufSize, ReplicationNum, BlockSize ); NUnit.Framework.Assert.IsTrue(dfs.dfs.Exists(fileStr)); // write bytes into the file. AppendTestUtil.Log.Info("size=" + size); stm.Write(buffer, 0, size); string originalLeaseHolder = NameNodeAdapter.GetLeaseHolderForPath(cluster.GetNameNode (), fileStr); NUnit.Framework.Assert.IsFalse("original lease holder should not be the NN", originalLeaseHolder .Equals(HdfsServerConstants.NamenodeLeaseHolder)); // hflush file AppendTestUtil.Log.Info("hflush"); stm.Hflush(); // check visible length HdfsDataInputStream @in = (HdfsDataInputStream)dfs.Open(filePath); NUnit.Framework.Assert.AreEqual(size, @in.GetVisibleLength()); @in.Close(); if (doRename) { fileStr += ".renamed"; Path renamedPath = new Path(fileStr); NUnit.Framework.Assert.IsTrue(dfs.Rename(filePath, renamedPath)); filePath = renamedPath; } // kill the lease renewal thread AppendTestUtil.Log.Info("leasechecker.interruptAndJoin()"); dfs.dfs.GetLeaseRenewer().InterruptAndJoin(); // Make sure the DNs don't send a heartbeat for a while, so the blocks // won't actually get completed during lease recovery. foreach (DataNode dn in cluster.GetDataNodes()) { DataNodeTestUtils.SetHeartbeatsDisabledForTests(dn, true); } // set the hard limit to be 1 second cluster.SetLeasePeriod(LongLeasePeriod, ShortLeasePeriod); // Make sure lease recovery begins. Sharpen.Thread.Sleep(HdfsServerConstants.NamenodeLeaseRecheckInterval * 2); CheckLease(fileStr, size); cluster.RestartNameNode(false); CheckLease(fileStr, size); // Let the DNs send heartbeats again. foreach (DataNode dn_1 in cluster.GetDataNodes()) { DataNodeTestUtils.SetHeartbeatsDisabledForTests(dn_1, false); } cluster.WaitActive(); // set the hard limit to be 1 second, to initiate lease recovery. cluster.SetLeasePeriod(LongLeasePeriod, ShortLeasePeriod); // wait for lease recovery to complete LocatedBlocks locatedBlocks; do { Sharpen.Thread.Sleep(ShortLeasePeriod); locatedBlocks = dfs.dfs.GetLocatedBlocks(fileStr, 0L, size); }while (locatedBlocks.IsUnderConstruction()); NUnit.Framework.Assert.AreEqual(size, locatedBlocks.GetFileLength()); // make sure that the client can't write data anymore. try { stm.Write('b'); stm.Hflush(); NUnit.Framework.Assert.Fail("Should not be able to flush after we've lost the lease" ); } catch (IOException e) { Log.Info("Expceted exception on write/hflush", e); } try { stm.Close(); NUnit.Framework.Assert.Fail("Should not be able to close after we've lost the lease" ); } catch (IOException e) { Log.Info("Expected exception on close", e); } // verify data AppendTestUtil.Log.Info("File size is good. Now validating sizes from datanodes..." ); AppendTestUtil.CheckFullFile(dfs, filePath, size, buffer, fileStr); }