/// <summary>Convert from a JSON file</summary> /// <param name="resource">input file</param> /// <returns>the parsed JSON</returns> /// <exception cref="System.IO.IOException">IO problems</exception> /// <exception cref="Org.Codehaus.Jackson.Map.JsonMappingException">failure to map from the JSON to this class /// </exception> /// <exception cref="Org.Codehaus.Jackson.JsonParseException"/> public virtual T FromResource(string resource) { lock (this) { InputStream resStream = null; try { resStream = this.GetType().GetResourceAsStream(resource); if (resStream == null) { throw new FileNotFoundException(resource); } return(mapper.ReadValue(resStream, classType)); } catch (IOException e) { Log.Error("Exception while parsing json resource {}: {}", resource, e); throw; } finally { IOUtils.CloseStream(resStream); } } }
/// <summary> /// Test which randomly alternates between appending with /// CRC32 and with CRC32C, crossing several block boundaries. /// </summary> /// <remarks> /// Test which randomly alternates between appending with /// CRC32 and with CRC32C, crossing several block boundaries. /// Then, checks that all of the data can be read back correct. /// </remarks> /// <exception cref="System.IO.IOException"/> public virtual void TestAlgoSwitchRandomized() { FileSystem fsWithCrc32 = CreateFsWithChecksum("CRC32", 512); FileSystem fsWithCrc32C = CreateFsWithChecksum("CRC32C", 512); Path p = new Path("/testAlgoSwitchRandomized"); long seed = Time.Now(); System.Console.Out.WriteLine("seed: " + seed); Random r = new Random(seed); // Create empty to start IOUtils.CloseStream(fsWithCrc32.Create(p)); long st = Time.Now(); int len = 0; while (Time.Now() - st < RandomTestRuntime) { int thisLen = r.Next(500); FileSystem fs = (r.NextBoolean() ? fsWithCrc32 : fsWithCrc32C); FSDataOutputStream stm = fs.Append(p); try { AppendTestUtil.Write(stm, len, thisLen); } finally { stm.Close(); } len += thisLen; } AppendTestUtil.Check(fsWithCrc32, p, len); AppendTestUtil.Check(fsWithCrc32C, p, len); }
/// <exception cref="System.IO.IOException"/> private void TestRbwReplicas(MiniDFSCluster cluster, bool isCorrupt) { FSDataOutputStream @out = null; FileSystem fs = cluster.GetFileSystem(); Path src = new Path("/test.txt"); try { int fileLen = 515; // create some rbw replicas on disk byte[] writeBuf = new byte[fileLen]; new Random().NextBytes(writeBuf); @out = fs.Create(src); @out.Write(writeBuf); @out.Hflush(); DataNode dn = cluster.GetDataNodes()[0]; foreach (FsVolumeSpi v in Dataset(dn).GetVolumes()) { FsVolumeImpl volume = (FsVolumeImpl)v; FilePath currentDir = volume.GetCurrentDir().GetParentFile().GetParentFile(); FilePath rbwDir = new FilePath(currentDir, "rbw"); foreach (FilePath file in rbwDir.ListFiles()) { if (isCorrupt && Block.IsBlockFilename(file)) { new RandomAccessFile(file, "rw").SetLength(fileLen - 1); } } } // corrupt cluster.RestartDataNodes(); cluster.WaitActive(); dn = cluster.GetDataNodes()[0]; // check volumeMap: one rwr replica string bpid = cluster.GetNamesystem().GetBlockPoolId(); ReplicaMap replicas = Dataset(dn).volumeMap; NUnit.Framework.Assert.AreEqual(1, replicas.Size(bpid)); ReplicaInfo replica = replicas.Replicas(bpid).GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.ReplicaState.Rwr, replica.GetState ()); if (isCorrupt) { NUnit.Framework.Assert.AreEqual((fileLen - 1) / 512 * 512, replica.GetNumBytes()); } else { NUnit.Framework.Assert.AreEqual(fileLen, replica.GetNumBytes()); } Dataset(dn).Invalidate(bpid, new Block[] { replica }); } finally { IOUtils.CloseStream(@out); if (fs.Exists(src)) { fs.Delete(src, false); } fs.Close(); } }
/// <summary>Read checksum into given buffer</summary> /// <param name="buf">buffer to read the checksum into</param> /// <param name="checksumOffset">offset at which to write the checksum into buf</param> /// <param name="checksumLen">length of checksum to write</param> /// <exception cref="System.IO.IOException">on error</exception> private void ReadChecksum(byte[] buf, int checksumOffset, int checksumLen) { if (checksumSize <= 0 && checksumIn == null) { return; } try { checksumIn.ReadFully(buf, checksumOffset, checksumLen); } catch (IOException e) { Log.Warn(" Could not read or failed to veirfy checksum for data" + " at offset " + offset + " for block " + block, e); IOUtils.CloseStream(checksumIn); checksumIn = null; if (corruptChecksumOk) { if (checksumOffset < checksumLen) { // Just fill the array with zeros. Arrays.Fill(buf, checksumOffset, checksumLen, unchecked ((byte)0)); } } else { throw; } } }
/// <summary> /// The idea for making sure that there is no more than one instance /// running in an HDFS is to create a file in the HDFS, writes the hostname /// of the machine on which the instance is running to the file, but did not /// close the file until it exits. /// </summary> /// <remarks> /// The idea for making sure that there is no more than one instance /// running in an HDFS is to create a file in the HDFS, writes the hostname /// of the machine on which the instance is running to the file, but did not /// close the file until it exits. /// This prevents the second instance from running because it can not /// creates the file while the first one is running. /// This method checks if there is any running instance. If no, mark yes. /// Note that this is an atomic operation. /// </remarks> /// <returns> /// null if there is a running instance; /// otherwise, the output stream to the newly created file. /// </returns> /// <exception cref="System.IO.IOException"/> private OutputStream CheckAndMarkRunning() { try { if (fs.Exists(idPath)) { // try appending to it so that it will fail fast if another balancer is // running. IOUtils.CloseStream(fs.Append(idPath)); fs.Delete(idPath, true); } FSDataOutputStream fsout = fs.Create(idPath, false); // mark balancer idPath to be deleted during filesystem closure fs.DeleteOnExit(idPath); if (write2IdFile) { fsout.WriteBytes(Sharpen.Runtime.GetLocalHost().GetHostName()); fsout.Hflush(); } return(fsout); } catch (RemoteException e) { if (typeof(AlreadyBeingCreatedException).FullName.Equals(e.GetClassName())) { return(null); } else { throw; } } }
/// <exception cref="System.Exception"/> public virtual void TestRmWithNonexistentGlob() { Configuration conf = new Configuration(); FsShell shell = new FsShell(); shell.SetConf(conf); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); TextWriter err = new TextWriter(bytes); TextWriter oldErr = System.Console.Error; Runtime.SetErr(err); string results; try { int exit = shell.Run(new string[] { "-rm", "nomatch*" }); Assert.Equal(1, exit); results = bytes.ToString(); Assert.True(results.Contains("rm: `nomatch*': No such file or directory" )); } finally { IOUtils.CloseStream(err); Runtime.SetErr(oldErr); } }
private static Credentials ConvertCredentialsFromByteBuffer(ByteBuffer appAttemptTokens ) { DataInputByteBuffer dibb = new DataInputByteBuffer(); try { Credentials credentials = null; if (appAttemptTokens != null) { credentials = new Credentials(); appAttemptTokens.Rewind(); dibb.Reset(appAttemptTokens); credentials.ReadTokenStorageStream(dibb); } return(credentials); } catch (IOException) { Log.Error("Failed to convert Credentials from ByteBuffer."); System.Diagnostics.Debug.Assert(false); return(null); } finally { IOUtils.CloseStream(dibb); } }
public virtual void TestOpenFileTwice() { Describe("verify that two opened file streams are independent"); Path path = Path("testopenfiletwice.txt"); byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255); //this file now has a simple rule: offset => value ContractTestUtils.CreateFile(GetFileSystem(), path, false, block); //open first FSDataInputStream instream1 = GetFileSystem().Open(path); int c = instream1.Read(); Assert.Equal(0, c); FSDataInputStream instream2 = null; try { instream2 = GetFileSystem().Open(path); Assert.Equal("first read of instream 2", 0, instream2.Read()); Assert.Equal("second read of instream 1", 1, instream1.Read()); instream1.Close(); Assert.Equal("second read of instream 2", 1, instream2.Read()); //close instream1 again instream1.Close(); } finally { IOUtils.CloseStream(instream1); IOUtils.CloseStream(instream2); } }
protected internal VersionInfo(string component) { info = new Properties(); string versionInfoFile = component + "-version-info.properties"; InputStream @is = null; try { @is = Thread.CurrentThread().GetContextClassLoader().GetResourceAsStream( versionInfoFile); if (@is == null) { throw new IOException("Resource not found"); } info.Load(@is); } catch (IOException ex) { LogFactory.GetLog(GetType()).Warn("Could not read '" + versionInfoFile + "', " + ex.ToString(), ex); } finally { IOUtils.CloseStream(@is); } }
public virtual void TestAppendWithPipelineRecovery() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; FSDataOutputStream @out = null; try { cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(true).ManageNameDfsDirs (true).NumDataNodes(4).Racks(new string[] { "/rack1", "/rack1", "/rack2", "/rack2" }).Build(); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); Path path = new Path("/test1"); @out = fs.Create(path, true, BlockSize, (short)3, BlockSize); AppendTestUtil.Write(@out, 0, 1024); @out.Close(); cluster.StopDataNode(3); @out = fs.Append(path); AppendTestUtil.Write(@out, 1024, 1024); @out.Close(); cluster.RestartNameNode(true); AppendTestUtil.Check(fs, path, 2048); } finally { IOUtils.CloseStream(@out); if (null != cluster) { cluster.Shutdown(); } } }
/// <summary> /// Used on Windows to determine if the specified file is a symlink that /// targets a directory. /// </summary> /// <remarks> /// Used on Windows to determine if the specified file is a symlink that /// targets a directory. On most platforms, these checks can be done using /// commons-io. On Windows, the commons-io implementation is unreliable and /// always returns false. Instead, this method checks the output of the dir /// command. After migrating to Java 7, this method can be removed in favor /// of the new method java.nio.file.Files.isSymbolicLink, which is expected to /// work cross-platform. /// </remarks> /// <param name="file">File to check</param> /// <returns>boolean true if the file is a symlink that targets a directory</returns> /// <exception cref="System.IO.IOException">thrown for any I/O error</exception> private static bool IsWindowsSymlinkedDirectory(FilePath file) { string dirOut = Shell.ExecCommand("cmd", "/c", "dir", file.GetAbsoluteFile().GetParent ()); StringReader sr = new StringReader(dirOut); BufferedReader br = new BufferedReader(sr); try { string line = br.ReadLine(); while (line != null) { line = br.ReadLine(); if (line.Contains(file.GetName()) && line.Contains("<SYMLINKD>")) { return(true); } } return(false); } finally { IOUtils.CloseStream(br); IOUtils.CloseStream(sr); } }
public virtual void TestBlockReportsWhileFileBeingWritten() { FSDataOutputStream @out = fs.Create(TestFilePath); try { AppendTestUtil.Write(@out, 0, 10); @out.Hflush(); // Block report will include the RBW replica, but will be // queued on the StandbyNode. cluster.TriggerBlockReports(); } finally { IOUtils.CloseStream(@out); } cluster.TransitionToStandby(0); cluster.TransitionToActive(1); // Verify that no replicas are marked corrupt, and that the // file is readable from the failed-over standby. BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager()); BlockManagerTestUtil.UpdateState(nn2.GetNamesystem().GetBlockManager()); NUnit.Framework.Assert.AreEqual(0, nn1.GetNamesystem().GetCorruptReplicaBlocks()); NUnit.Framework.Assert.AreEqual(0, nn2.GetNamesystem().GetCorruptReplicaBlocks()); DFSTestUtil.ReadFile(fs, TestFilePath); }
/// <exception cref="System.IO.IOException"/> public static int WriteCompressedByteArray(BinaryWriter writer, byte[] bytes) { if (bytes != null) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); GZIPOutputStream gzout = new GZIPOutputStream(bos); try { gzout.Write(bytes, 0, bytes.Length); gzout.Close(); gzout = null; } finally { IOUtils.CloseStream(gzout); } byte[] buffer = bos.ToByteArray(); int len = buffer.Length; @out.WriteInt(len); @out.Write(buffer, 0, len); /* debug only! Once we have confidence, can lose this. */ return((bytes.Length != 0) ? (100 * buffer.Length) / bytes.Length : 0); } else { @out.WriteInt(-1); return(-1); } }
/// <summary> /// Test for the case where the client beings to read a long block, but doesn't /// read bytes off the stream quickly. /// </summary> /// <remarks> /// Test for the case where the client beings to read a long block, but doesn't /// read bytes off the stream quickly. The datanode should time out sending the /// chunks and the transceiver should die, even if it has a long keepalive. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestSlowReader() { // Set a client socket cache expiry time much longer than // the datanode-side expiration time. long ClientExpiryMs = 600000L; Configuration clientConf = new Configuration(conf); clientConf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, ClientExpiryMs ); clientConf.Set(DFSConfigKeys.DfsClientContext, "testSlowReader"); DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI() , clientConf); // Restart the DN with a shorter write timeout. MiniDFSCluster.DataNodeProperties props = cluster.StopDataNode(0); props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketWriteTimeoutKey, WriteTimeout); props.conf.SetInt(DFSConfigKeys.DfsDatanodeSocketReuseKeepaliveKey, 120000); NUnit.Framework.Assert.IsTrue(cluster.RestartDataNode(props, true)); dn = cluster.GetDataNodes()[0]; // Wait for heartbeats to avoid a startup race where we // try to write the block while the DN is still starting. cluster.TriggerHeartbeats(); DFSTestUtil.CreateFile(fs, TestFile, 1024 * 1024 * 8L, (short)1, 0L); FSDataInputStream stm = fs.Open(TestFile); stm.Read(); AssertXceiverCount(1); GenericTestUtils.WaitFor(new _Supplier_193(this), 500, 50000); // DN should time out in sendChunks, and this should force // the xceiver to exit. IOUtils.CloseStream(stm); }
// Set up fault injection mock. /// <summary> /// Run through the creation of a log without any faults injected, /// and count how many RPCs are made to each node. /// </summary> /// <remarks> /// Run through the creation of a log without any faults injected, /// and count how many RPCs are made to each node. This sets the /// bounds for the other test cases, so they can exhaustively explore /// the space of potential failures. /// </remarks> /// <exception cref="System.Exception"/> private static long DetermineMaxIpcNumber() { Configuration conf = new Configuration(); MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).Build(); QuorumJournalManager qjm = null; long ret; try { qjm = CreateInjectableQJM(cluster); qjm.Format(QJMTestUtil.FakeNsinfo); DoWorkload(cluster, qjm); ICollection <int> ipcCounts = Sets.NewTreeSet(); foreach (AsyncLogger l in qjm.GetLoggerSetForTests().GetLoggersForTests()) { TestQJMWithFaults.InvocationCountingChannel ch = (TestQJMWithFaults.InvocationCountingChannel )l; ch.WaitForAllPendingCalls(); ipcCounts.AddItem(ch.GetRpcCount()); } // All of the loggers should have sent the same number of RPCs, since there // were no failures. NUnit.Framework.Assert.AreEqual(1, ipcCounts.Count); ret = ipcCounts.First(); Log.Info("Max IPC count = " + ret); } finally { IOUtils.CloseStream(qjm); cluster.Shutdown(); } return(ret); }
internal QueueConfigurationParser(string confFile, bool areAclsEnabled) { //Default root. //xml tags for mapred-queues.xml // The value read from queues config file for this tag is not used at all. // To enable queue acls and job acls, mapreduce.cluster.acls.enabled is // to be set in mapred-site.xml aclsEnabled = areAclsEnabled; FilePath file = new FilePath(confFile).GetAbsoluteFile(); if (!file.Exists()) { throw new RuntimeException("Configuration file not found at " + confFile); } InputStream @in = null; try { @in = new BufferedInputStream(new FileInputStream(file)); LoadFrom(@in); } catch (IOException ioe) { throw new RuntimeException(ioe); } finally { IOUtils.CloseStream(@in); } }
/// <summary> /// Corrupt the given VERSION file by replacing a given /// key with a new value and re-writing the file. /// </summary> /// <param name="versionFile">the VERSION file to corrupt</param> /// <param name="key">the key to replace</param> /// <param name="value">the new value for this key</param> /// <exception cref="System.IO.IOException"/> public static void CorruptVersionFile(FilePath versionFile, string key, string value ) { Properties props = new Properties(); FileInputStream fis = new FileInputStream(versionFile); FileOutputStream @out = null; try { props.Load(fis); IOUtils.CloseStream(fis); if (value == null || value.IsEmpty()) { props.Remove(key); } else { props.SetProperty(key, value); } @out = new FileOutputStream(versionFile); props.Store(@out, null); } finally { IOUtils.Cleanup(null, fis, @out); } }
private static ByteBuffer ConvertCredentialsToByteBuffer(Credentials credentials) { ByteBuffer appAttemptTokens = null; DataOutputBuffer dob = new DataOutputBuffer(); try { if (credentials != null) { credentials.WriteTokenStorageToStream(dob); appAttemptTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength()); } return(appAttemptTokens); } catch (IOException) { Log.Error("Failed to convert Credentials to ByteBuffer."); System.Diagnostics.Debug.Assert(false); return(null); } finally { IOUtils.CloseStream(dob); } }
/// <summary> /// Calculate the md5sum of an image after zeroing out the transaction ID /// field in the header. /// </summary> /// <remarks> /// Calculate the md5sum of an image after zeroing out the transaction ID /// field in the header. This is useful for tests that want to verify /// that two checkpoints have identical namespaces. /// </remarks> /// <exception cref="System.IO.IOException"/> public static string GetImageFileMD5IgnoringTxId(FilePath imageFile) { FilePath tmpFile = FilePath.CreateTempFile("hadoop_imagefile_tmp", "fsimage"); tmpFile.DeleteOnExit(); try { Files.Copy(imageFile, tmpFile); RandomAccessFile raf = new RandomAccessFile(tmpFile, "rw"); try { raf.Seek(ImageTxidPos); raf.WriteLong(0); } finally { IOUtils.CloseStream(raf); } return(GetFileMD5(tmpFile)); } finally { tmpFile.Delete(); } }
/// <summary>Assert that a set of properties files all contain the same data.</summary> /// <remarks> /// Assert that a set of properties files all contain the same data. /// We cannot simply check the md5sums here, since Properties files /// contain timestamps -- thus, two properties files from the same /// saveNamespace operation may actually differ in md5sum. /// </remarks> /// <param name="propFiles">the files to compare</param> /// <exception cref="System.IO.IOException">if the files cannot be opened or read</exception> /// <exception cref="System.Exception">if the files differ</exception> public static void AssertPropertiesFilesSame(FilePath[] propFiles) { ICollection <KeyValuePair <object, object> > prevProps = null; foreach (FilePath f in propFiles) { Properties props; FileInputStream @is = new FileInputStream(f); try { props = new Properties(); props.Load(@is); } finally { IOUtils.CloseStream(@is); } if (prevProps == null) { prevProps = props; } else { ICollection <KeyValuePair <object, object> > diff = Sets.SymmetricDifference(prevProps , props); if (!diff.IsEmpty()) { NUnit.Framework.Assert.Fail("Properties file " + f + " differs from " + propFiles [0]); } } } }
public override void Run() { FSDataOutputStream @out = null; int i = 0; try { @out = fs.Create(filepath); for (; running; i++) { System.Console.Out.WriteLine(GetName() + " writes " + i); @out.Write(i); @out.Hflush(); Sleep(100); } } catch (Exception e) { System.Console.Out.WriteLine(GetName() + " dies: e=" + e); } finally { System.Console.Out.WriteLine(GetName() + ": i=" + i); IOUtils.CloseStream(@out); } }
/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { FileInputStream editFileIn = null; try { ServletContext context = GetServletContext(); Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf ); string journalId = request.GetParameter(JournalIdParam); QuorumJournalManager.CheckJournalId(journalId); JNStorage storage = JournalNodeHttpServer.GetJournalFromContext(context, journalId ).GetStorage(); // Check security if (!CheckRequestorOrSendError(conf, request, response)) { return; } // Check that the namespace info is correct if (!CheckStorageInfoOrSendError(storage, request, response)) { return; } long segmentTxId = ServletUtil.ParseLongParam(request, SegmentTxidParam); FileJournalManager fjm = storage.GetJournalManager(); FilePath editFile; lock (fjm) { // Synchronize on the FJM so that the file doesn't get finalized // out from underneath us while we're in the process of opening // it up. FileJournalManager.EditLogFile elf = fjm.GetLogFile(segmentTxId); if (elf == null) { response.SendError(HttpServletResponse.ScNotFound, "No edit log found starting at txid " + segmentTxId); return; } editFile = elf.GetFile(); ImageServlet.SetVerificationHeadersForGet(response, editFile); ImageServlet.SetFileNameHeaders(response, editFile); editFileIn = new FileInputStream(editFile); } DataTransferThrottler throttler = ImageServlet.GetThrottler(conf); // send edits TransferFsImage.CopyFileToStream(response.GetOutputStream(), editFile, editFileIn , throttler); } catch (Exception t) { string errMsg = "getedit failed. " + StringUtils.StringifyException(t); response.SendError(HttpServletResponse.ScInternalServerError, errMsg); throw new IOException(errMsg); } finally { IOUtils.CloseStream(editFileIn); } }
public virtual void TestCreatedFileIsImmediatelyVisible() { Describe("verify that a newly created file exists as soon as open returns"); Path path = Path("testCreatedFileIsImmediatelyVisible"); FSDataOutputStream @out = null; try { @out = GetFileSystem().Create(path, false, 4096, (short)1, 1024); if (!GetFileSystem().Exists(path)) { if (IsSupported(IsBlobstore)) { // object store: downgrade to a skip so that the failure is visible // in test results ContractTestUtils.Skip("Filesystem is an object store and newly created files are not immediately visible" ); } AssertPathExists("expected path to be visible before anything written", path); } } finally { IOUtils.CloseStream(@out); } }
public virtual void TestGetSet() { BestEffortLongFile f = new BestEffortLongFile(File, 12345L); try { // Before the file exists, should return default. NUnit.Framework.Assert.AreEqual(12345L, f.Get()); // And first access should open it. NUnit.Framework.Assert.IsTrue(File.Exists()); Random r = new Random(); for (int i = 0; i < 100; i++) { long newVal = r.NextLong(); // Changing the value should be reflected in the next get() call. f.Set(newVal); NUnit.Framework.Assert.AreEqual(newVal, f.Get()); // And should be reflected in a new instance (ie it actually got // written to the file) BestEffortLongFile f2 = new BestEffortLongFile(File, 999L); try { NUnit.Framework.Assert.AreEqual(newVal, f2.Get()); } finally { IOUtils.CloseStream(f2); } } } finally { IOUtils.CloseStream(f); } }
/// <exception cref="System.IO.IOException"/> private void WriteToHostsFile(params string[] hosts) { if (!hostFile.Exists()) { TempDir.Mkdirs(); hostFile.CreateNewFile(); } FileOutputStream fStream = null; try { fStream = new FileOutputStream(hostFile); for (int i = 0; i < hosts.Length; i++) { fStream.Write(Sharpen.Runtime.GetBytesForString(hosts[i])); fStream.Write(Sharpen.Runtime.GetBytesForString("\n")); } } finally { if (fStream != null) { IOUtils.CloseStream(fStream); fStream = null; } } }
public static void TearDown() { IOUtils.CloseStream(dfs); if (cluster != null) { cluster.Shutdown(); } }
public virtual void TestRestartDfs() { Configuration conf = new HdfsConfiguration(); // Turn off persistent IPC, so that the DFSClient can survive NN restart conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0); MiniDFSCluster cluster = null; long len = 0; FSDataOutputStream stream; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); FileSystem fs = cluster.GetFileSystem(); // Creating a file with 4096 blockSize to write multiple blocks stream = fs.Create(FilePath, true, BlockSize, (short)1, BlockSize); stream.Write(DataBeforeRestart); stream.Hflush(); // Wait for at least a few blocks to get through while (len <= BlockSize) { FileStatus status = fs.GetFileStatus(FilePath); len = status.GetLen(); Sharpen.Thread.Sleep(100); } // explicitly do NOT close the file. cluster.RestartNameNode(); // Check that the file has no less bytes than before the restart // This would mean that blocks were successfully persisted to the log FileStatus status_1 = fs.GetFileStatus(FilePath); NUnit.Framework.Assert.IsTrue("Length too short: " + status_1.GetLen(), status_1. GetLen() >= len); // And keep writing (ensures that leases are also persisted correctly) stream.Write(DataAfterRestart); stream.Close(); // Verify that the data showed up, both from before and after the restart. FSDataInputStream readStream = fs.Open(FilePath); try { byte[] verifyBuf = new byte[DataBeforeRestart.Length]; IOUtils.ReadFully(readStream, verifyBuf, 0, verifyBuf.Length); Assert.AssertArrayEquals(DataBeforeRestart, verifyBuf); IOUtils.ReadFully(readStream, verifyBuf, 0, verifyBuf.Length); Assert.AssertArrayEquals(DataAfterRestart, verifyBuf); } finally { IOUtils.CloseStream(readStream); } } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> private void DoWriteOverFailoverTest(TestPipelinesFailover.TestScenario scenario, TestPipelinesFailover.MethodToTestIdempotence methodToTest) { Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize); // Don't check replication periodically. conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1000); FSDataOutputStream stm = null; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleHATopology()).NumDataNodes(3).Build(); try { int sizeWritten = 0; cluster.WaitActive(); cluster.TransitionToActive(0); Sharpen.Thread.Sleep(500); Log.Info("Starting with NN 0 active"); FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf); stm = fs.Create(TestPath); // write a block and a half AppendTestUtil.Write(stm, 0, BlockAndAHalf); sizeWritten += BlockAndAHalf; // Make sure all of the blocks are written out before failover. stm.Hflush(); Log.Info("Failing over to NN 1"); scenario.Run(cluster); // NOTE: explicitly do *not* make any further metadata calls // to the NN here. The next IPC call should be to allocate the next // block. Any other call would notice the failover and not test // idempotence of the operation (HDFS-3031) FSNamesystem ns1 = cluster.GetNameNode(1).GetNamesystem(); BlockManagerTestUtil.UpdateState(ns1.GetBlockManager()); NUnit.Framework.Assert.AreEqual(0, ns1.GetPendingReplicationBlocks()); NUnit.Framework.Assert.AreEqual(0, ns1.GetCorruptReplicaBlocks()); NUnit.Framework.Assert.AreEqual(0, ns1.GetMissingBlocksCount()); // If we're testing allocateBlock()'s idempotence, write another // block and a half, so we have to allocate a new block. // Otherise, don't write anything, so our next RPC will be // completeFile() if we're testing idempotence of that operation. if (methodToTest == TestPipelinesFailover.MethodToTestIdempotence.AllocateBlock) { // write another block and a half AppendTestUtil.Write(stm, sizeWritten, BlockAndAHalf); sizeWritten += BlockAndAHalf; } stm.Close(); stm = null; AppendTestUtil.Check(fs, TestPath, sizeWritten); } finally { IOUtils.CloseStream(stm); cluster.Shutdown(); } }
/// <summary>Retrieves the number of links to the specified file.</summary> /// <exception cref="System.IO.IOException"/> public static int GetLinkCount(FilePath fileName) { if (fileName == null) { throw new IOException("invalid argument to getLinkCount: file name is null"); } if (!fileName.Exists()) { throw new FileNotFoundException(fileName + " not found."); } // construct and execute shell command string[] cmd = getHardLinkCommand.LinkCount(fileName); string inpMsg = null; string errMsg = null; int exitValue = -1; BufferedReader @in = null; Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(cmd); try { shexec.Execute(); @in = new BufferedReader(new StringReader(shexec.GetOutput())); inpMsg = @in.ReadLine(); exitValue = shexec.GetExitCode(); if (inpMsg == null || exitValue != 0) { throw CreateIOException(fileName, inpMsg, errMsg, exitValue, null); } if (Shell.Solaris) { string[] result = inpMsg.Split("\\s+"); return(System.Convert.ToInt32(result[1])); } else { return(System.Convert.ToInt32(inpMsg)); } } catch (Shell.ExitCodeException e) { inpMsg = shexec.GetOutput(); errMsg = e.Message; exitValue = e.GetExitCode(); throw CreateIOException(fileName, inpMsg, errMsg, exitValue, e); } catch (FormatException e) { throw CreateIOException(fileName, inpMsg, errMsg, exitValue, e); } finally { IOUtils.CloseStream(@in); } }
/// <exception cref="System.IO.IOException"/> internal static FSEditLogLoader.EditLogValidation ScanEditLog(FilePath file) { Org.Apache.Hadoop.Hdfs.Server.Namenode.EditLogFileInputStream @in; try { @in = new Org.Apache.Hadoop.Hdfs.Server.Namenode.EditLogFileInputStream(file); // read the header, initialize the inputstream, but do not check the // layoutversion @in.GetVersion(false); } catch (EditLogFileInputStream.LogHeaderCorruptException e) { Log.Warn("Log file " + file + " has no valid header", e); return(new FSEditLogLoader.EditLogValidation(0, HdfsConstants.InvalidTxid, true)); } long lastPos = 0; long lastTxId = HdfsConstants.InvalidTxid; long numValid = 0; try { while (true) { long txid = HdfsConstants.InvalidTxid; lastPos = @in.GetPosition(); try { if ((txid = @in.ScanNextOp()) == HdfsConstants.InvalidTxid) { break; } } catch (Exception t) { FSImage.Log.Warn("Caught exception after scanning through " + numValid + " ops from " + @in + " while determining its valid length. Position was " + lastPos, t); @in.Resync(); FSImage.Log.Warn("After resync, position is " + @in.GetPosition()); continue; } if (lastTxId == HdfsConstants.InvalidTxid || txid > lastTxId) { lastTxId = txid; } numValid++; } return(new FSEditLogLoader.EditLogValidation(lastPos, lastTxId, false)); } finally { IOUtils.CloseStream(@in); } }