/// <summary> /// Transfers data from FileChannel using /// <see cref="FileChannel.TransferTo(long, long, WritableByteChannel) /// "/> /// . /// Updates <code>waitForWritableTime</code> and <code>transferToTime</code> /// with the time spent blocked on the network and the time spent transferring /// data from disk to network respectively. /// Similar to readFully(), this waits till requested amount of /// data is transfered. /// </summary> /// <param name="fileCh">FileChannel to transfer data from.</param> /// <param name="position">position within the channel where the transfer begins</param> /// <param name="count">number of bytes to transfer.</param> /// <param name="waitForWritableTime"> /// nanoseconds spent waiting for the socket /// to become writable /// </param> /// <param name="transferTime">nanoseconds spent transferring data</param> /// <exception cref="System.IO.EOFException"> /// /// If end of input file is reached before requested number of /// bytes are transfered. /// </exception> /// <exception cref="SocketTimeoutException"> /// /// If this channel blocks transfer longer than timeout for /// this stream. /// </exception> /// <exception cref="System.IO.IOException"> /// Includes any exception thrown by /// <see cref="FileChannel.TransferTo(long, long, WritableByteChannel) /// "/> /// . /// </exception> public virtual void TransferToFully(FileChannel fileCh, long position, int count, LongWritable waitForWritableTime, LongWritable transferToTime) { long waitTime = 0; long transferTime = 0; while (count > 0) { /* * Ideally we should wait after transferTo returns 0. But because of * a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988), * which throws an exception instead of returning 0, we wait for the * channel to be writable before writing to it. If you ever see * IOException with message "Resource temporarily unavailable" * thrown here, please let us know. * * Once we move to JAVA SE 7, wait should be moved to correct place. */ long start = Runtime.NanoTime(); WaitForWritable(); long wait = Runtime.NanoTime(); int nTransfered = (int)fileCh.TransferTo(position, count, GetChannel()); if (nTransfered == 0) { //check if end of file is reached. if (position >= fileCh.Size()) { throw new EOFException("EOF Reached. file size is " + fileCh.Size() + " and " + count + " more bytes left to be " + "transfered."); } } else { //otherwise assume the socket is full. //waitForWritable(); // see comment above. if (nTransfered < 0) { throw new IOException("Unexpected return of " + nTransfered + " from transferTo()" ); } else { position += nTransfered; count -= nTransfered; } } long transfer = Runtime.NanoTime(); waitTime += wait - start; transferTime += transfer - wait; } if (waitForWritableTime != null) { waitForWritableTime.Set(waitTime); } if (transferToTime != null) { transferToTime.Set(transferTime); } }
/// <exception cref="System.IO.IOException"/> private void Preallocate() { long position = fc.Position(); long size = fc.Size(); int bufSize = doubleBuf.GetReadyBuf().GetLength(); long need = bufSize - (size - position); if (need <= 0) { return; } long oldSize = size; long total = 0; long fillCapacity = fill.Capacity(); while (need > 0) { fill.Position(0); IOUtils.WriteFully(fc, fill, size); need -= fillCapacity; size += fillCapacity; total += fillCapacity; } if (Log.IsDebugEnabled()) { Log.Debug("Preallocated " + total + " bytes at the end of " + "the edit log (offset " + oldSize + ")"); } }
static long CopyFileStream(FileInputStream @is, FileOutputStream os) { FileChannel srcChannel = null; FileChannel destChannel = null; long length; try { srcChannel = @is.Channel; destChannel = os.Channel; length = srcChannel.TransferTo(0L, srcChannel.Size(), destChannel); } finally { if (srcChannel != null) { srcChannel.Close(); } if (destChannel != null) { destChannel.Close(); } } return(length); }
/// <summary> /// Return the length of bytes in the given file after subtracting /// the trailer of 0xFF (OP_INVALID)s. /// </summary> /// <remarks> /// Return the length of bytes in the given file after subtracting /// the trailer of 0xFF (OP_INVALID)s. /// This seeks to the end of the file and reads chunks backwards until /// it finds a non-0xFF byte. /// </remarks> /// <exception cref="System.IO.IOException">if the file cannot be read</exception> private static long GetNonTrailerLength(FilePath f) { int chunkSizeToRead = 256 * 1024; FileInputStream fis = new FileInputStream(f); try { byte[] buf = new byte[chunkSizeToRead]; FileChannel fc = fis.GetChannel(); long size = fc.Size(); long pos = size - (size % chunkSizeToRead); while (pos >= 0) { fc.Position(pos); int readLen = (int)Math.Min(size - pos, chunkSizeToRead); IOUtils.ReadFully(fis, buf, 0, readLen); for (int i = readLen - 1; i >= 0; i--) { if (buf[i] != FSEditLogOpCodes.OpInvalid.GetOpCode()) { return(pos + i + 1); } } // + 1 since we count this byte! pos -= chunkSizeToRead; } return(0); } finally { fis.Close(); } }
/// <exception cref="System.Exception"/> public virtual void TestMlock() { Assume.AssumeTrue(NativeIO.IsAvailable()); FilePath TestFile = new FilePath(new FilePath(Runtime.GetProperty("test.build.data" , "build/test/data")), "testMlockFile"); int BufLen = 12289; byte[] buf = new byte[BufLen]; int bufSum = 0; for (int i = 0; i < buf.Length; i++) { buf[i] = unchecked ((byte)(i % 60)); bufSum += buf[i]; } FileOutputStream fos = new FileOutputStream(TestFile); try { fos.Write(buf); fos.GetChannel().Force(true); } finally { fos.Close(); } FileInputStream fis = null; FileChannel channel = null; try { // Map file into memory fis = new FileInputStream(TestFile); channel = fis.GetChannel(); long fileSize = channel.Size(); MappedByteBuffer mapbuf = channel.Map(FileChannel.MapMode.ReadOnly, 0, fileSize); // mlock the buffer NativeIO.POSIX.Mlock(mapbuf, fileSize); // Read the buffer int sum = 0; for (int i_1 = 0; i_1 < fileSize; i_1++) { sum += mapbuf.Get(i_1); } Assert.Equal("Expected sums to be equal", bufSum, sum); // munmap the buffer, which also implicitly unlocks it NativeIO.POSIX.Munmap(mapbuf); } finally { if (channel != null) { channel.Close(); } if (fis != null) { fis.Close(); } } }
public virtual void TestRecoveryMode() { // edits generated by nnHelper (MiniDFSCluster), should have all op codes // binary, XML, reparsed binary string edits = nnHelper.GenerateEdits(); FileOutputStream os = new FileOutputStream(edits, true); // Corrupt the file by truncating the end FileChannel editsFile = os.GetChannel(); editsFile.Truncate(editsFile.Size() - 5); string editsParsedXml = folder.NewFile("editsRecoveredParsed.xml").GetAbsolutePath (); string editsReparsed = folder.NewFile("editsRecoveredReparsed").GetAbsolutePath(); string editsParsedXml2 = folder.NewFile("editsRecoveredParsed2.xml").GetAbsolutePath (); // Can't read the corrupted file without recovery mode NUnit.Framework.Assert.AreEqual(-1, RunOev(edits, editsParsedXml, "xml", false)); // parse to XML then back to binary NUnit.Framework.Assert.AreEqual(0, RunOev(edits, editsParsedXml, "xml", true)); NUnit.Framework.Assert.AreEqual(0, RunOev(editsParsedXml, editsReparsed, "binary" , false)); NUnit.Framework.Assert.AreEqual(0, RunOev(editsReparsed, editsParsedXml2, "xml", false)); // judgment time NUnit.Framework.Assert.IsTrue("Test round trip", FileUtils.ContentEqualsIgnoreEOL (new FilePath(editsParsedXml), new FilePath(editsParsedXml2), "UTF-8")); os.Close(); }
/// <summary>Copy the current file content into the temporary file.</summary> /// <remarks> /// Copy the current file content into the temporary file. /// <p> /// This method saves the current file content by inserting it into the /// temporary file, so that the caller can safely append rather than replace /// the primary file. /// <p> /// This method does nothing if the current file does not exist, or exists /// but is empty. /// </remarks> /// <exception cref="System.IO.IOException"> /// the temporary file could not be written, or a read error /// occurred while reading from the current file. The lock is /// released before throwing the underlying IO exception to the /// caller. /// </exception> /// <exception cref="Sharpen.RuntimeException"> /// the temporary file could not be written. The lock is released /// before throwing the underlying exception to the caller. /// </exception> public virtual void CopyCurrentContent() { RequireLock(); try { FileInputStream fis = new FileInputStream(@ref); try { if (fsync) { FileChannel @in = fis.GetChannel(); long pos = 0; long cnt = @in.Size(); while (0 < cnt) { long r = os.GetChannel().TransferFrom(@in, pos, cnt); pos += r; cnt -= r; } } else { byte[] buf = new byte[2048]; int r; while ((r = fis.Read(buf)) >= 0) { os.Write(buf, 0, r); } } } finally { fis.Close(); } } catch (FileNotFoundException) { } catch (IOException ioe) { // Don't worry about a file that doesn't exist yet, it // conceptually has no current content to copy. // Unlock(); throw; } catch (RuntimeException ioe) { Unlock(); throw; } catch (Error ioe) { Unlock(); throw; } }
/// <exception cref="System.IO.IOException"/> private int ReadWithoutBounceBuffer(ByteBuffer buf) { lock (this) { FreeDataBufIfExists(); FreeChecksumBufIfExists(); int total = 0; while (buf.HasRemaining()) { int nRead = dataIn.Read(buf, dataPos); if (nRead <= 0) { break; } dataPos += nRead; total += nRead; } return((total == 0 && (dataPos == dataIn.Size())) ? -1 : total); } }
/// <summary>Creates output buffers and file object.</summary> /// <param name="conf">Configuration object</param> /// <param name="name">File name to store edit log</param> /// <param name="size">Size of flush buffer</param> /// <exception cref="System.IO.IOException"/> public EditLogFileOutputStream(Configuration conf, FilePath name, int size) : base() { shouldSyncWritesAndSkipFsync = conf.GetBoolean(DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflush , DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflushDefault); file = name; doubleBuf = new EditsDoubleBuffer(size); RandomAccessFile rp; if (shouldSyncWritesAndSkipFsync) { rp = new RandomAccessFile(name, "rws"); } else { rp = new RandomAccessFile(name, "rw"); } fp = new FileOutputStream(rp.GetFD()); // open for append fc = rp.GetChannel(); fc.Position(fc.Size()); }
/** * 文件拷贝 * * @param src 源文件 * @param desc 目的文件 */ public static void fileChannelCopy(File src, File desc) { //createFile(src); createFile(desc); FileInputStream fi = null; FileOutputStream fo = null; try { fi = new FileInputStream(src); fo = new FileOutputStream(desc); FileChannel inStream = fi.Channel; //得到对应的文件通道 FileChannel outStream = fo.Channel; //得到对应的文件通道 inStream.TransferTo(0, inStream.Size(), outStream); //连接两个通道,并且从in通道读取,然后写入out通道 } catch (IOException e) { e.PrintStackTrace(); } finally { try { if (fo != null) { fo.Close(); } if (fi != null) { fi.Close(); } } catch (IOException e) { e.PrintStackTrace(); } } }
/// <summary> /// Unbuffered file copy from src to dst without tainting OS buffer cache /// In POSIX platform: /// It uses FileChannel#transferTo() which internally attempts /// unbuffered IO on OS with native sendfile64() support and falls back to /// buffered IO otherwise. /// </summary> /// <remarks> /// Unbuffered file copy from src to dst without tainting OS buffer cache /// In POSIX platform: /// It uses FileChannel#transferTo() which internally attempts /// unbuffered IO on OS with native sendfile64() support and falls back to /// buffered IO otherwise. /// It minimizes the number of FileChannel#transferTo call by passing the the /// src file size directly instead of a smaller size as the 3rd parameter. /// This saves the number of sendfile64() system call when native sendfile64() /// is supported. In the two fall back cases where sendfile is not supported, /// FileChannle#transferTo already has its own batching of size 8 MB and 8 KB, /// respectively. /// In Windows Platform: /// It uses its own native wrapper of CopyFileEx with COPY_FILE_NO_BUFFERING /// flag, which is supported on Windows Server 2008 and above. /// Ideally, we should use FileChannel#transferTo() across both POSIX and Windows /// platform. Unfortunately, the wrapper(Java_sun_nio_ch_FileChannelImpl_transferTo0) /// used by FileChannel#transferTo for unbuffered IO is not implemented on Windows. /// Based on OpenJDK 6/7/8 source code, Java_sun_nio_ch_FileChannelImpl_transferTo0 /// on Windows simply returns IOS_UNSUPPORTED. /// Note: This simple native wrapper does minimal parameter checking before copy and /// consistency check (e.g., size) after copy. /// It is recommended to use wrapper function like /// the Storage#nativeCopyFileUnbuffered() function in hadoop-hdfs with pre/post copy /// checks. /// </remarks> /// <param name="src">The source path</param> /// <param name="dst">The destination path</param> /// <exception cref="System.IO.IOException"/> public static void CopyFileUnbuffered(FilePath src, FilePath dst) { if (nativeLoaded && Shell.Windows) { CopyFileUnbuffered0(src.GetAbsolutePath(), dst.GetAbsolutePath()); } else { FileInputStream fis = null; FileOutputStream fos = null; FileChannel input = null; FileChannel output = null; try { fis = new FileInputStream(src); fos = new FileOutputStream(dst); input = fis.GetChannel(); output = fos.GetChannel(); long remaining = input.Size(); long position = 0; long transferred = 0; while (remaining > 0) { transferred = input.TransferTo(position, remaining, output); remaining -= transferred; position += transferred; } } finally { IOUtils.Cleanup(Log, output); IOUtils.Cleanup(Log, fos); IOUtils.Cleanup(Log, input); IOUtils.Cleanup(Log, fis); } } }
/// <exception cref="System.Exception"/> private static long[] GetBlockSizes(HdfsBlockLocation[] locs) { long[] sizes = new long[locs.Length]; for (int i = 0; i < locs.Length; i++) { HdfsBlockLocation loc = locs[i]; string bpid = loc.GetLocatedBlock().GetBlock().GetBlockPoolId(); Block block = loc.GetLocatedBlock().GetBlock().GetLocalBlock(); ExtendedBlock extBlock = new ExtendedBlock(bpid, block); FileInputStream blockInputStream = null; FileChannel blockChannel = null; try { blockInputStream = (FileInputStream)fsd.GetBlockInputStream(extBlock, 0); blockChannel = blockInputStream.GetChannel(); sizes[i] = blockChannel.Size(); } finally { IOUtils.Cleanup(Log, blockChannel, blockInputStream); } } return(sizes); }
internal virtual MappedByteBuffer LoadMmapInternal() { try { FileChannel channel = dataStream.GetChannel(); MappedByteBuffer mmap = channel.Map(FileChannel.MapMode.ReadOnly, 0, Math.Min(int.MaxValue , channel.Size())); if (Log.IsTraceEnabled()) { Log.Trace(this + ": created mmap of size " + channel.Size()); } return(mmap); } catch (IOException e) { Log.Warn(this + ": mmap error", e); return(null); } catch (RuntimeException e) { Log.Warn(this + ": mmap error", e); return(null); } }
private void CopyFileToDownloads(Android.Net.Uri croppedFileUri) { try { String downloadsDirectoryPath = Android.OS.Environment.GetExternalStoragePublicDirectory(Android.OS.Environment.DirectoryDownloads).AbsolutePath; String filename = String.Format("%d_%s", Calendar.Instance.TimeInMillis, croppedFileUri.LastPathSegment); Java.IO.File saveFile = new Java.IO.File(downloadsDirectoryPath, filename); FileInputStream inStream = new FileInputStream(new Java.IO.File(croppedFileUri.Path)); FileOutputStream outStream = new FileOutputStream(saveFile); FileChannel inChannel = inStream.Channel; FileChannel outChannel = outStream.Channel; inChannel.TransferTo(0, inChannel.Size(), outChannel); inStream.Close(); outStream.Close(); ShowNotification(saveFile); } catch (Exception e) { Toast.MakeText(this, e.Message, ToastLength.Short).Show(); } }
/// <summary>check if nn.getCorruptFiles() returns a file that has corrupted blocks</summary> /// <exception cref="System.Exception"/> public virtual void TestListCorruptFilesCorruptedBlock() { MiniDFSCluster cluster = null; Random random = new Random(); try { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsDatanodeDirectoryscanIntervalKey, 1); // datanode scans directories conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 3 * 1000); // datanode sends block reports // Set short retry timeouts so this test runs faster conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10); cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); // create two files with one block each DFSTestUtil util = new DFSTestUtil.Builder().SetName("testCorruptFilesCorruptedBlock" ).SetNumFiles(2).SetMaxLevels(1).SetMaxSize(512).Build(); util.CreateFiles(fs, "/srcdat10"); // fetch bad file list from namenode. There should be none. NameNode namenode = cluster.GetNameNode(); ICollection <FSNamesystem.CorruptFileBlockInfo> badFiles = namenode.GetNamesystem( ).ListCorruptFileBlocks("/", null); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " corrupt files. Expecting None." , badFiles.Count == 0); // Now deliberately corrupt one block string bpid = cluster.GetNamesystem().GetBlockPoolId(); FilePath storageDir = cluster.GetInstanceStorageDir(0, 1); FilePath data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid); NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists()); IList <FilePath> metaFiles = MiniDFSCluster.GetAllBlockMetadataFiles(data_dir); NUnit.Framework.Assert.IsTrue("Data directory does not contain any blocks or there was an " + "IO error", metaFiles != null && !metaFiles.IsEmpty()); FilePath metaFile = metaFiles[0]; RandomAccessFile file = new RandomAccessFile(metaFile, "rw"); FileChannel channel = file.GetChannel(); long position = channel.Size() - 2; int length = 2; byte[] buffer = new byte[length]; random.NextBytes(buffer); channel.Write(ByteBuffer.Wrap(buffer), position); file.Close(); Log.Info("Deliberately corrupting file " + metaFile.GetName() + " at offset " + position + " length " + length); // read all files to trigger detection of corrupted replica try { util.CheckFiles(fs, "/srcdat10"); } catch (BlockMissingException) { System.Console.Out.WriteLine("Received BlockMissingException as expected."); } catch (IOException e) { NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly. Expecting BlockMissingException " + " but received IOException " + e, false); } // fetch bad file list from namenode. There should be one file. badFiles = namenode.GetNamesystem().ListCorruptFileBlocks("/", null); Log.Info("Namenode has bad files. " + badFiles.Count); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting 1." , badFiles.Count == 1); util.Cleanup(fs, "/srcdat10"); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>check if DFS can handle corrupted CRC blocks</summary> /// <exception cref="System.Exception"/> private void Thistest(Configuration conf, DFSTestUtil util) { MiniDFSCluster cluster = null; int numDataNodes = 2; short replFactor = 2; Random random = new Random(); // Set short retry timeouts so this test runs faster conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); util.CreateFiles(fs, "/srcdat", replFactor); util.WaitReplication(fs, "/srcdat", (short)2); // Now deliberately remove/truncate meta blocks from the first // directory of the first datanode. The complete absense of a meta // file disallows this Datanode to send data to another datanode. // However, a client is alowed access to this block. // FilePath storageDir = cluster.GetInstanceStorageDir(0, 1); string bpid = cluster.GetNamesystem().GetBlockPoolId(); FilePath data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid); NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists()); FilePath[] blocks = data_dir.ListFiles(); NUnit.Framework.Assert.IsTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.Length > 0)); int num = 0; for (int idx = 0; idx < blocks.Length; idx++) { if (blocks[idx].GetName().StartsWith(Block.BlockFilePrefix) && blocks[idx].GetName ().EndsWith(".meta")) { num++; if (num % 3 == 0) { // // remove .meta file // System.Console.Out.WriteLine("Deliberately removing file " + blocks[idx].GetName( )); NUnit.Framework.Assert.IsTrue("Cannot remove file.", blocks[idx].Delete()); } else { if (num % 3 == 1) { // // shorten .meta file // RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw"); FileChannel channel = file.GetChannel(); int newsize = random.Next((int)channel.Size() / 2); System.Console.Out.WriteLine("Deliberately truncating file " + blocks[idx].GetName () + " to size " + newsize + " bytes."); channel.Truncate(newsize); file.Close(); } else { // // corrupt a few bytes of the metafile // RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw"); FileChannel channel = file.GetChannel(); long position = 0; // // The very first time, corrupt the meta header at offset 0 // if (num != 2) { position = (long)random.Next((int)channel.Size()); } int length = random.Next((int)(channel.Size() - position + 1)); byte[] buffer = new byte[length]; random.NextBytes(buffer); channel.Write(ByteBuffer.Wrap(buffer), position); System.Console.Out.WriteLine("Deliberately corrupting file " + blocks[idx].GetName () + " at offset " + position + " length " + length); file.Close(); } } } } // // Now deliberately corrupt all meta blocks from the second // directory of the first datanode // storageDir = cluster.GetInstanceStorageDir(0, 1); data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid); NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists()); blocks = data_dir.ListFiles(); NUnit.Framework.Assert.IsTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.Length > 0)); int count = 0; FilePath previous = null; for (int idx_1 = 0; idx_1 < blocks.Length; idx_1++) { if (blocks[idx_1].GetName().StartsWith("blk_") && blocks[idx_1].GetName().EndsWith (".meta")) { // // Move the previous metafile into the current one. // count++; if (count % 2 == 0) { System.Console.Out.WriteLine("Deliberately insertimg bad crc into files " + blocks [idx_1].GetName() + " " + previous.GetName()); NUnit.Framework.Assert.IsTrue("Cannot remove file.", blocks[idx_1].Delete()); NUnit.Framework.Assert.IsTrue("Cannot corrupt meta file.", previous.RenameTo(blocks [idx_1])); NUnit.Framework.Assert.IsTrue("Cannot recreate empty meta file.", previous.CreateNewFile ()); previous = null; } else { previous = blocks[idx_1]; } } } // // Only one replica is possibly corrupted. The other replica should still // be good. Verify. // NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly.", util.CheckFiles (fs, "/srcdat")); System.Console.Out.WriteLine("All File still have a valid replica"); // // set replication factor back to 1. This causes only one replica of // of each block to remain in HDFS. The check is to make sure that // the corrupted replica generated above is the one that gets deleted. // This test is currently disabled until HADOOP-1557 is solved. // util.SetReplication(fs, "/srcdat", (short)1); //util.waitReplication(fs, "/srcdat", (short)1); //System.out.println("All Files done with removing replicas"); //assertTrue("Excess replicas deleted. Corrupted replicas found.", // util.checkFiles(fs, "/srcdat")); System.Console.Out.WriteLine("The excess-corrupted-replica test is disabled " + " pending HADOOP-1557" ); util.Cleanup(fs, "/srcdat"); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Check that listCorruptFileBlocks works while the namenode is still in safemode. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestListCorruptFileBlocksInSafeMode() { MiniDFSCluster cluster = null; Random random = new Random(); try { Configuration conf = new HdfsConfiguration(); // datanode scans directories conf.SetInt(DFSConfigKeys.DfsDatanodeDirectoryscanIntervalKey, 1); // datanode sends block reports conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 3 * 1000); // never leave safemode automatically conf.SetFloat(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, 1.5f); // start populating repl queues immediately conf.SetFloat(DFSConfigKeys.DfsNamenodeReplQueueThresholdPctKey, 0f); // Set short retry timeouts so this test runs faster conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10); cluster = new MiniDFSCluster.Builder(conf).WaitSafeMode(false).Build(); cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false); FileSystem fs = cluster.GetFileSystem(); // create two files with one block each DFSTestUtil util = new DFSTestUtil.Builder().SetName("testListCorruptFileBlocksInSafeMode" ).SetNumFiles(2).SetMaxLevels(1).SetMaxSize(512).Build(); util.CreateFiles(fs, "/srcdat10"); // fetch bad file list from namenode. There should be none. ICollection <FSNamesystem.CorruptFileBlockInfo> badFiles = cluster.GetNameNode().GetNamesystem ().ListCorruptFileBlocks("/", null); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " corrupt files. Expecting None." , badFiles.Count == 0); // Now deliberately corrupt one block FilePath storageDir = cluster.GetInstanceStorageDir(0, 0); FilePath data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, cluster.GetNamesystem ().GetBlockPoolId()); NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists()); IList <FilePath> metaFiles = MiniDFSCluster.GetAllBlockMetadataFiles(data_dir); NUnit.Framework.Assert.IsTrue("Data directory does not contain any blocks or there was an " + "IO error", metaFiles != null && !metaFiles.IsEmpty()); FilePath metaFile = metaFiles[0]; RandomAccessFile file = new RandomAccessFile(metaFile, "rw"); FileChannel channel = file.GetChannel(); long position = channel.Size() - 2; int length = 2; byte[] buffer = new byte[length]; random.NextBytes(buffer); channel.Write(ByteBuffer.Wrap(buffer), position); file.Close(); Log.Info("Deliberately corrupting file " + metaFile.GetName() + " at offset " + position + " length " + length); // read all files to trigger detection of corrupted replica try { util.CheckFiles(fs, "/srcdat10"); } catch (BlockMissingException) { System.Console.Out.WriteLine("Received BlockMissingException as expected."); } catch (IOException e) { NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly. " + "Expecting BlockMissingException " + " but received IOException " + e, false); } // fetch bad file list from namenode. There should be one file. badFiles = cluster.GetNameNode().GetNamesystem().ListCorruptFileBlocks("/", null); Log.Info("Namenode has bad files. " + badFiles.Count); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting 1." , badFiles.Count == 1); // restart namenode cluster.RestartNameNode(0); fs = cluster.GetFileSystem(); // wait until replication queues have been initialized while (!cluster.GetNameNode().namesystem.IsPopulatingReplQueues()) { try { Log.Info("waiting for replication queues"); Sharpen.Thread.Sleep(1000); } catch (Exception) { } } // read all files to trigger detection of corrupted replica try { util.CheckFiles(fs, "/srcdat10"); } catch (BlockMissingException) { System.Console.Out.WriteLine("Received BlockMissingException as expected."); } catch (IOException e) { NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly. " + "Expecting BlockMissingException " + " but received IOException " + e, false); } // fetch bad file list from namenode. There should be one file. badFiles = cluster.GetNameNode().GetNamesystem().ListCorruptFileBlocks("/", null); Log.Info("Namenode has bad files. " + badFiles.Count); NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting 1." , badFiles.Count == 1); // check that we are still in safe mode NUnit.Framework.Assert.IsTrue("Namenode is not in safe mode", cluster.GetNameNode ().IsInSafeMode()); // now leave safe mode so that we can clean up cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false); util.Cleanup(fs, "/srcdat10"); } catch (Exception e) { Log.Error(StringUtils.StringifyException(e)); throw; } finally { if (cluster != null) { cluster.Shutdown(); } } }
public static void LoadDict(Context context) { try { bool resaveEntries = false; dictParts = new List <byte[]>(); dictIndexes = new List <int>(); File dictFd = new File(context.FilesDir, "dict.db"); if (!dictFd.Exists()) { // || dictFd.length() != 4961308) { System.Console.WriteLine("DOES NOT EXIST!!!!!"); CopyFile(context, "dict.db"); dictFd = new File(context.FilesDir, "dict.db"); resaveEntries = true; } dictFile = new RandomAccessFile(dictFd, "r"); File idxFd = new File(context.FilesDir, "idx.db"); if (!idxFd.Exists()) { // || idxFd.length() != 3145553) { CopyFile(context, "idx.db"); idxFd = new File(context.FilesDir, "idx.db"); resaveEntries = true; } FileInputStream idxBuf = new FileInputStream(idxFd); if (!new File(context.FilesDir, "entries.bin").Exists() || !new File(context.FilesDir, "parts.bin").Exists()) { resaveEntries = true; } entries = IntBuffer.Allocate(1649830); int index = 0; //System.Console.WriteLine("LoadDict STEP 1"); if (idxBuf != null) { int readLen, offset = 0, partLen = 200000; byte[] dictPart = new byte[partLen]; int totalRead = 0; int totalLen = (int)idxFd.Length(); while (totalRead < totalLen && (readLen = idxBuf.Read(dictPart, offset, dictPart.Length - offset)) > 0) { //System.Console.WriteLine("LoadDict \ntotalRead = " + totalRead + "\ntotalLen = " + totalLen + "\nreadLen = " + readLen + "\nidxBuf.Read = " + idxBuf.Read(dictPart, offset, dictPart.Length - offset)); totalRead += readLen; int j = offset + readLen - 1; byte[] newDictPart = null; if (readLen == partLen - offset) { //System.Console.WriteLine("LoadDict STEP 4.1 " + dictPart[j] + " :: j => " + j); while (dictPart[j] > 0) { //System.Console.WriteLine("j = " + j + "\ndictPart[j] = " + dictPart[j]); j--; } //System.Console.WriteLine("LoadDict STEP 4.2"); while (dictPart[j] < 0) { System.Console.WriteLine("j = " + j); j--; } //System.Console.WriteLine("LoadDict STEP 4.3"); offset = partLen - j - 1; //System.Console.WriteLine("LoadDict STEP 4.4"); newDictPart = new byte[Math.Min(totalLen - totalRead + offset, partLen)]; //System.Console.WriteLine("LoadDict STEP 4.5"); Java.Lang.JavaSystem.Arraycopy(dictPart, j + 1, newDictPart, 0, offset); //Array.Copy(dictPart, j + 1, newDictPart, 0, offset); } else { offset = 0; } //System.Console.WriteLine("LoadDict STEP 5"); if (resaveEntries) { dictIndexes.Add(index); //System.Console.WriteLine("LoadDict STEP 6"); int i = 0; while (i <= j) { entries.Put(index++, i); while (i <= j && dictPart[i] < 0) { i++; } while (i <= j && dictPart[i] >= 0) { i++; } } } //System.Console.WriteLine("LoadDict STEP 7"); dictParts.Add(dictPart); dictPart = newDictPart; //System.Console.WriteLine("LoadDict STEP 8"); } idxBuf.Close(); } if (resaveEntries) { //System.Console.WriteLine("LoadDict STEP 9"); DataOutputStream entriesOut = null, partsOut = null; //System.Console.WriteLine("LoadDict STEP 10"); entriesOut = new DataOutputStream(context.OpenFileOutput("entries.bin", FileCreationMode.Private)); int count = entries.Capacity(); for (int i = 0; i < count; i++) { entriesOut.WriteInt(entries.Get(i)); } //System.Console.WriteLine("LoadDict STEP 11"); partsOut = new DataOutputStream(context.OpenFileOutput("parts.bin", FileCreationMode.Private)); foreach (int i in dictIndexes) { partsOut.WriteInt(i); } //System.Console.WriteLine("LoadDict STEP 12"); if (entriesOut != null) { entriesOut.Flush(); entriesOut.Close(); } if (partsOut != null) { partsOut.Flush(); partsOut.Close(); } } else { //System.Console.WriteLine("LoadDict NOW RESAVING ENTRIES"); string documentpath = System.Environment.GetFolderPath(System.Environment.SpecialFolder.Personal); Java.IO.File sdpath = global::Android.OS.Environment.GetExternalStoragePublicDirectory(global::Android.OS.Environment.DirectoryDownloads); File entriesDB = new File(documentpath, "entries.bin"); File partsDB = new File(documentpath, "parts.bin"); FileInputStream entriesIn = null, partsIn = null; //entriesIn = context.OpenFileInput("entries.bin"); entriesIn = new FileInputStream(entriesDB); //entriesIn = new FileInputStream(new File("entries.bin")); FileChannel file = entriesIn.Channel; ByteBuffer bb = ByteBuffer.Allocate(4 * 1649830); file.Read(bb); bb.Rewind(); entries = bb.AsIntBuffer(); file.Close(); partsIn = new FileInputStream(partsDB); //partsIn = new FileInputStream(new File("parts.bin")); //partsIn = (context.OpenFileInput("parts.bin"); file = partsIn.Channel; bb = ByteBuffer.Allocate((int)file.Size()); file.Read(bb); bb.Rewind(); IntBuffer ib = bb.AsIntBuffer(); int count = ib.Capacity(); //System.Console.WriteLine("LoadDict STEP 99 " + count); for (int i = 0; i < count; i++) { dictIndexes.Add(ib.Get(i)); } file.Close(); if (entriesIn != null) { entriesIn.Close(); } if (partsIn != null) { partsIn.Close(); } } } catch (Exception e) { System.Console.WriteLine("Dict LoadDict ERROR => " + e.Message); Log.Equals("chinesreader", e.Message); } byteBuffer = new byte[1090]; sharedPrefs = PreferenceManager.GetDefaultSharedPreferences(context); }