/// <summary>
/// Transfers data from FileChannel using
/// <see cref="FileChannel.TransferTo(long, long, WritableByteChannel)
/// "/>
/// .
/// Updates <code>waitForWritableTime</code> and <code>transferToTime</code>
/// with the time spent blocked on the network and the time spent transferring
/// data from disk to network respectively.
/// Similar to readFully(), this waits till requested amount of
/// data is transfered.
/// </summary>
/// <param name="fileCh">FileChannel to transfer data from.</param>
/// <param name="position">position within the channel where the transfer begins</param>
/// <param name="count">number of bytes to transfer.</param>
/// <param name="waitForWritableTime">
/// nanoseconds spent waiting for the socket
/// to become writable
/// </param>
/// <param name="transferTime">nanoseconds spent transferring data</param>
/// <exception cref="System.IO.EOFException">
///
/// If end of input file is reached before requested number of
/// bytes are transfered.
/// </exception>
/// <exception cref="SocketTimeoutException">
///
/// If this channel blocks transfer longer than timeout for
/// this stream.
/// </exception>
/// <exception cref="System.IO.IOException">
/// Includes any exception thrown by
/// <see cref="FileChannel.TransferTo(long, long, WritableByteChannel)
/// "/>
/// .
/// </exception>
public virtual void TransferToFully(FileChannel fileCh, long position, int count,
LongWritable waitForWritableTime, LongWritable transferToTime)
{
long waitTime = 0;
long transferTime = 0;
while (count > 0)
{
/*
* Ideally we should wait after transferTo returns 0. But because of
* a bug in JRE on Linux (http://bugs.sun.com/view_bug.do?bug_id=5103988),
* which throws an exception instead of returning 0, we wait for the
* channel to be writable before writing to it. If you ever see
* IOException with message "Resource temporarily unavailable"
* thrown here, please let us know.
*
* Once we move to JAVA SE 7, wait should be moved to correct place.
*/
long start = Runtime.NanoTime();
WaitForWritable();
long wait = Runtime.NanoTime();
int nTransfered = (int)fileCh.TransferTo(position, count, GetChannel());
if (nTransfered == 0)
{
//check if end of file is reached.
if (position >= fileCh.Size())
{
throw new EOFException("EOF Reached. file size is " + fileCh.Size() + " and " + count
+ " more bytes left to be " + "transfered.");
}
}
else
{
//otherwise assume the socket is full.
//waitForWritable(); // see comment above.
if (nTransfered < 0)
{
throw new IOException("Unexpected return of " + nTransfered + " from transferTo()"
);
}
else
{
position += nTransfered;
count -= nTransfered;
}
}
long transfer = Runtime.NanoTime();
waitTime += wait - start;
transferTime += transfer - wait;
}
if (waitForWritableTime != null)
{
waitForWritableTime.Set(waitTime);
}
if (transferToTime != null)
{
transferToTime.Set(transferTime);
}
}
/// <exception cref="System.IO.IOException"/>
private void Preallocate()
{
long position = fc.Position();
long size = fc.Size();
int bufSize = doubleBuf.GetReadyBuf().GetLength();
long need = bufSize - (size - position);
if (need <= 0)
{
return;
}
long oldSize = size;
long total = 0;
long fillCapacity = fill.Capacity();
while (need > 0)
{
fill.Position(0);
IOUtils.WriteFully(fc, fill, size);
need -= fillCapacity;
size += fillCapacity;
total += fillCapacity;
}
if (Log.IsDebugEnabled())
{
Log.Debug("Preallocated " + total + " bytes at the end of " + "the edit log (offset "
+ oldSize + ")");
}
}
static long CopyFileStream(FileInputStream @is, FileOutputStream os)
{
FileChannel srcChannel = null;
FileChannel destChannel = null;
long length;
try
{
srcChannel = @is.Channel;
destChannel = os.Channel;
length = srcChannel.TransferTo(0L, srcChannel.Size(), destChannel);
}
finally
{
if (srcChannel != null)
{
srcChannel.Close();
}
if (destChannel != null)
{
destChannel.Close();
}
}
return(length);
}
/// <summary>
/// Return the length of bytes in the given file after subtracting
/// the trailer of 0xFF (OP_INVALID)s.
/// </summary>
/// <remarks>
/// Return the length of bytes in the given file after subtracting
/// the trailer of 0xFF (OP_INVALID)s.
/// This seeks to the end of the file and reads chunks backwards until
/// it finds a non-0xFF byte.
/// </remarks>
/// <exception cref="System.IO.IOException">if the file cannot be read</exception>
private static long GetNonTrailerLength(FilePath f)
{
int chunkSizeToRead = 256 * 1024;
FileInputStream fis = new FileInputStream(f);
try
{
byte[] buf = new byte[chunkSizeToRead];
FileChannel fc = fis.GetChannel();
long size = fc.Size();
long pos = size - (size % chunkSizeToRead);
while (pos >= 0)
{
fc.Position(pos);
int readLen = (int)Math.Min(size - pos, chunkSizeToRead);
IOUtils.ReadFully(fis, buf, 0, readLen);
for (int i = readLen - 1; i >= 0; i--)
{
if (buf[i] != FSEditLogOpCodes.OpInvalid.GetOpCode())
{
return(pos + i + 1);
}
}
// + 1 since we count this byte!
pos -= chunkSizeToRead;
}
return(0);
}
finally
{
fis.Close();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestMlock()
{
Assume.AssumeTrue(NativeIO.IsAvailable());
FilePath TestFile = new FilePath(new FilePath(Runtime.GetProperty("test.build.data"
, "build/test/data")), "testMlockFile");
int BufLen = 12289;
byte[] buf = new byte[BufLen];
int bufSum = 0;
for (int i = 0; i < buf.Length; i++)
{
buf[i] = unchecked ((byte)(i % 60));
bufSum += buf[i];
}
FileOutputStream fos = new FileOutputStream(TestFile);
try
{
fos.Write(buf);
fos.GetChannel().Force(true);
}
finally
{
fos.Close();
}
FileInputStream fis = null;
FileChannel channel = null;
try
{
// Map file into memory
fis = new FileInputStream(TestFile);
channel = fis.GetChannel();
long fileSize = channel.Size();
MappedByteBuffer mapbuf = channel.Map(FileChannel.MapMode.ReadOnly, 0, fileSize);
// mlock the buffer
NativeIO.POSIX.Mlock(mapbuf, fileSize);
// Read the buffer
int sum = 0;
for (int i_1 = 0; i_1 < fileSize; i_1++)
{
sum += mapbuf.Get(i_1);
}
Assert.Equal("Expected sums to be equal", bufSum, sum);
// munmap the buffer, which also implicitly unlocks it
NativeIO.POSIX.Munmap(mapbuf);
}
finally
{
if (channel != null)
{
channel.Close();
}
if (fis != null)
{
fis.Close();
}
}
}
public virtual void TestRecoveryMode()
{
// edits generated by nnHelper (MiniDFSCluster), should have all op codes
// binary, XML, reparsed binary
string edits = nnHelper.GenerateEdits();
FileOutputStream os = new FileOutputStream(edits, true);
// Corrupt the file by truncating the end
FileChannel editsFile = os.GetChannel();
editsFile.Truncate(editsFile.Size() - 5);
string editsParsedXml = folder.NewFile("editsRecoveredParsed.xml").GetAbsolutePath
();
string editsReparsed = folder.NewFile("editsRecoveredReparsed").GetAbsolutePath();
string editsParsedXml2 = folder.NewFile("editsRecoveredParsed2.xml").GetAbsolutePath
();
// Can't read the corrupted file without recovery mode
NUnit.Framework.Assert.AreEqual(-1, RunOev(edits, editsParsedXml, "xml", false));
// parse to XML then back to binary
NUnit.Framework.Assert.AreEqual(0, RunOev(edits, editsParsedXml, "xml", true));
NUnit.Framework.Assert.AreEqual(0, RunOev(editsParsedXml, editsReparsed, "binary"
, false));
NUnit.Framework.Assert.AreEqual(0, RunOev(editsReparsed, editsParsedXml2, "xml",
false));
// judgment time
NUnit.Framework.Assert.IsTrue("Test round trip", FileUtils.ContentEqualsIgnoreEOL
(new FilePath(editsParsedXml), new FilePath(editsParsedXml2), "UTF-8"));
os.Close();
}
/// <summary>Copy the current file content into the temporary file.</summary>
/// <remarks>
/// Copy the current file content into the temporary file.
/// <p>
/// This method saves the current file content by inserting it into the
/// temporary file, so that the caller can safely append rather than replace
/// the primary file.
/// <p>
/// This method does nothing if the current file does not exist, or exists
/// but is empty.
/// </remarks>
/// <exception cref="System.IO.IOException">
/// the temporary file could not be written, or a read error
/// occurred while reading from the current file. The lock is
/// released before throwing the underlying IO exception to the
/// caller.
/// </exception>
/// <exception cref="Sharpen.RuntimeException">
/// the temporary file could not be written. The lock is released
/// before throwing the underlying exception to the caller.
/// </exception>
public virtual void CopyCurrentContent()
{
RequireLock();
try
{
FileInputStream fis = new FileInputStream(@ref);
try
{
if (fsync)
{
FileChannel @in = fis.GetChannel();
long pos = 0;
long cnt = @in.Size();
while (0 < cnt)
{
long r = os.GetChannel().TransferFrom(@in, pos, cnt);
pos += r;
cnt -= r;
}
}
else
{
byte[] buf = new byte[2048];
int r;
while ((r = fis.Read(buf)) >= 0)
{
os.Write(buf, 0, r);
}
}
}
finally
{
fis.Close();
}
}
catch (FileNotFoundException)
{
}
catch (IOException ioe)
{
// Don't worry about a file that doesn't exist yet, it
// conceptually has no current content to copy.
//
Unlock();
throw;
}
catch (RuntimeException ioe)
{
Unlock();
throw;
}
catch (Error ioe)
{
Unlock();
throw;
}
}
/// <exception cref="System.IO.IOException"/>
private int ReadWithoutBounceBuffer(ByteBuffer buf)
{
lock (this)
{
FreeDataBufIfExists();
FreeChecksumBufIfExists();
int total = 0;
while (buf.HasRemaining())
{
int nRead = dataIn.Read(buf, dataPos);
if (nRead <= 0)
{
break;
}
dataPos += nRead;
total += nRead;
}
return((total == 0 && (dataPos == dataIn.Size())) ? -1 : total);
}
}
/// <summary>Creates output buffers and file object.</summary>
/// <param name="conf">Configuration object</param>
/// <param name="name">File name to store edit log</param>
/// <param name="size">Size of flush buffer</param>
/// <exception cref="System.IO.IOException"/>
public EditLogFileOutputStream(Configuration conf, FilePath name, int size)
: base()
{
shouldSyncWritesAndSkipFsync = conf.GetBoolean(DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflush
, DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflushDefault);
file = name;
doubleBuf = new EditsDoubleBuffer(size);
RandomAccessFile rp;
if (shouldSyncWritesAndSkipFsync)
{
rp = new RandomAccessFile(name, "rws");
}
else
{
rp = new RandomAccessFile(name, "rw");
}
fp = new FileOutputStream(rp.GetFD());
// open for append
fc = rp.GetChannel();
fc.Position(fc.Size());
}
/**
* 文件拷贝
*
* @param src 源文件
* @param desc 目的文件
*/
public static void fileChannelCopy(File src, File desc)
{
//createFile(src);
createFile(desc);
FileInputStream fi = null;
FileOutputStream fo = null;
try
{
fi = new FileInputStream(src);
fo = new FileOutputStream(desc);
FileChannel inStream = fi.Channel; //得到对应的文件通道
FileChannel outStream = fo.Channel; //得到对应的文件通道
inStream.TransferTo(0, inStream.Size(), outStream); //连接两个通道，并且从in通道读取，然后写入out通道
}
catch (IOException e)
{
e.PrintStackTrace();
}
finally
{
try
{
if (fo != null)
{
fo.Close();
}
if (fi != null)
{
fi.Close();
}
}
catch (IOException e)
{
e.PrintStackTrace();
}
}
}
/// <summary>
/// Unbuffered file copy from src to dst without tainting OS buffer cache
/// In POSIX platform:
/// It uses FileChannel#transferTo() which internally attempts
/// unbuffered IO on OS with native sendfile64() support and falls back to
/// buffered IO otherwise.
/// </summary>
/// <remarks>
/// Unbuffered file copy from src to dst without tainting OS buffer cache
/// In POSIX platform:
/// It uses FileChannel#transferTo() which internally attempts
/// unbuffered IO on OS with native sendfile64() support and falls back to
/// buffered IO otherwise.
/// It minimizes the number of FileChannel#transferTo call by passing the the
/// src file size directly instead of a smaller size as the 3rd parameter.
/// This saves the number of sendfile64() system call when native sendfile64()
/// is supported. In the two fall back cases where sendfile is not supported,
/// FileChannle#transferTo already has its own batching of size 8 MB and 8 KB,
/// respectively.
/// In Windows Platform:
/// It uses its own native wrapper of CopyFileEx with COPY_FILE_NO_BUFFERING
/// flag, which is supported on Windows Server 2008 and above.
/// Ideally, we should use FileChannel#transferTo() across both POSIX and Windows
/// platform. Unfortunately, the wrapper(Java_sun_nio_ch_FileChannelImpl_transferTo0)
/// used by FileChannel#transferTo for unbuffered IO is not implemented on Windows.
/// Based on OpenJDK 6/7/8 source code, Java_sun_nio_ch_FileChannelImpl_transferTo0
/// on Windows simply returns IOS_UNSUPPORTED.
/// Note: This simple native wrapper does minimal parameter checking before copy and
/// consistency check (e.g., size) after copy.
/// It is recommended to use wrapper function like
/// the Storage#nativeCopyFileUnbuffered() function in hadoop-hdfs with pre/post copy
/// checks.
/// </remarks>
/// <param name="src">The source path</param>
/// <param name="dst">The destination path</param>
/// <exception cref="System.IO.IOException"/>
public static void CopyFileUnbuffered(FilePath src, FilePath dst)
{
if (nativeLoaded && Shell.Windows)
{
CopyFileUnbuffered0(src.GetAbsolutePath(), dst.GetAbsolutePath());
}
else
{
FileInputStream fis = null;
FileOutputStream fos = null;
FileChannel input = null;
FileChannel output = null;
try
{
fis = new FileInputStream(src);
fos = new FileOutputStream(dst);
input = fis.GetChannel();
output = fos.GetChannel();
long remaining = input.Size();
long position = 0;
long transferred = 0;
while (remaining > 0)
{
transferred = input.TransferTo(position, remaining, output);
remaining -= transferred;
position += transferred;
}
}
finally
{
IOUtils.Cleanup(Log, output);
IOUtils.Cleanup(Log, fos);
IOUtils.Cleanup(Log, input);
IOUtils.Cleanup(Log, fis);
}
}
}
/// <exception cref="System.Exception"/>
private static long[] GetBlockSizes(HdfsBlockLocation[] locs)
{
long[] sizes = new long[locs.Length];
for (int i = 0; i < locs.Length; i++)
{
HdfsBlockLocation loc = locs[i];
string bpid = loc.GetLocatedBlock().GetBlock().GetBlockPoolId();
Block block = loc.GetLocatedBlock().GetBlock().GetLocalBlock();
ExtendedBlock extBlock = new ExtendedBlock(bpid, block);
FileInputStream blockInputStream = null;
FileChannel blockChannel = null;
try
{
blockInputStream = (FileInputStream)fsd.GetBlockInputStream(extBlock, 0);
blockChannel = blockInputStream.GetChannel();
sizes[i] = blockChannel.Size();
}
finally
{
IOUtils.Cleanup(Log, blockChannel, blockInputStream);
}
}
return(sizes);
}
internal virtual MappedByteBuffer LoadMmapInternal()
{
try
{
FileChannel channel = dataStream.GetChannel();
MappedByteBuffer mmap = channel.Map(FileChannel.MapMode.ReadOnly, 0, Math.Min(int.MaxValue
, channel.Size()));
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": created mmap of size " + channel.Size());
}
return(mmap);
}
catch (IOException e)
{
Log.Warn(this + ": mmap error", e);
return(null);
}
catch (RuntimeException e)
{
Log.Warn(this + ": mmap error", e);
return(null);
}
}
private void CopyFileToDownloads(Android.Net.Uri croppedFileUri)
{
try
{
String downloadsDirectoryPath = Android.OS.Environment.GetExternalStoragePublicDirectory(Android.OS.Environment.DirectoryDownloads).AbsolutePath;
String filename = String.Format("%d_%s", Calendar.Instance.TimeInMillis, croppedFileUri.LastPathSegment);
Java.IO.File saveFile = new Java.IO.File(downloadsDirectoryPath, filename);
FileInputStream inStream = new FileInputStream(new Java.IO.File(croppedFileUri.Path));
FileOutputStream outStream = new FileOutputStream(saveFile);
FileChannel inChannel = inStream.Channel;
FileChannel outChannel = outStream.Channel;
inChannel.TransferTo(0, inChannel.Size(), outChannel);
inStream.Close();
outStream.Close();
ShowNotification(saveFile);
}
catch (Exception e)
{
Toast.MakeText(this, e.Message, ToastLength.Short).Show();
}
}
/// <summary>check if nn.getCorruptFiles() returns a file that has corrupted blocks</summary>
/// <exception cref="System.Exception"/>
public virtual void TestListCorruptFilesCorruptedBlock()
{
MiniDFSCluster cluster = null;
Random random = new Random();
try
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsDatanodeDirectoryscanIntervalKey, 1);
// datanode scans directories
conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 3 * 1000);
// datanode sends block reports
// Set short retry timeouts so this test runs faster
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
// create two files with one block each
DFSTestUtil util = new DFSTestUtil.Builder().SetName("testCorruptFilesCorruptedBlock"
).SetNumFiles(2).SetMaxLevels(1).SetMaxSize(512).Build();
util.CreateFiles(fs, "/srcdat10");
// fetch bad file list from namenode. There should be none.
NameNode namenode = cluster.GetNameNode();
ICollection <FSNamesystem.CorruptFileBlockInfo> badFiles = namenode.GetNamesystem(
).ListCorruptFileBlocks("/", null);
NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " corrupt files. Expecting None."
, badFiles.Count == 0);
// Now deliberately corrupt one block
string bpid = cluster.GetNamesystem().GetBlockPoolId();
FilePath storageDir = cluster.GetInstanceStorageDir(0, 1);
FilePath data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid);
NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists());
IList <FilePath> metaFiles = MiniDFSCluster.GetAllBlockMetadataFiles(data_dir);
NUnit.Framework.Assert.IsTrue("Data directory does not contain any blocks or there was an "
+ "IO error", metaFiles != null && !metaFiles.IsEmpty());
FilePath metaFile = metaFiles[0];
RandomAccessFile file = new RandomAccessFile(metaFile, "rw");
FileChannel channel = file.GetChannel();
long position = channel.Size() - 2;
int length = 2;
byte[] buffer = new byte[length];
random.NextBytes(buffer);
channel.Write(ByteBuffer.Wrap(buffer), position);
file.Close();
Log.Info("Deliberately corrupting file " + metaFile.GetName() + " at offset " + position
+ " length " + length);
// read all files to trigger detection of corrupted replica
try
{
util.CheckFiles(fs, "/srcdat10");
}
catch (BlockMissingException)
{
System.Console.Out.WriteLine("Received BlockMissingException as expected.");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly. Expecting BlockMissingException "
+ " but received IOException " + e, false);
}
// fetch bad file list from namenode. There should be one file.
badFiles = namenode.GetNamesystem().ListCorruptFileBlocks("/", null);
Log.Info("Namenode has bad files. " + badFiles.Count);
NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting 1."
, badFiles.Count == 1);
util.Cleanup(fs, "/srcdat10");
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>check if DFS can handle corrupted CRC blocks</summary>
/// <exception cref="System.Exception"/>
private void Thistest(Configuration conf, DFSTestUtil util)
{
MiniDFSCluster cluster = null;
int numDataNodes = 2;
short replFactor = 2;
Random random = new Random();
// Set short retry timeouts so this test runs faster
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
util.CreateFiles(fs, "/srcdat", replFactor);
util.WaitReplication(fs, "/srcdat", (short)2);
// Now deliberately remove/truncate meta blocks from the first
// directory of the first datanode. The complete absense of a meta
// file disallows this Datanode to send data to another datanode.
// However, a client is alowed access to this block.
//
FilePath storageDir = cluster.GetInstanceStorageDir(0, 1);
string bpid = cluster.GetNamesystem().GetBlockPoolId();
FilePath data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid);
NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists());
FilePath[] blocks = data_dir.ListFiles();
NUnit.Framework.Assert.IsTrue("Blocks do not exist in data-dir", (blocks != null) &&
(blocks.Length > 0));
int num = 0;
for (int idx = 0; idx < blocks.Length; idx++)
{
if (blocks[idx].GetName().StartsWith(Block.BlockFilePrefix) && blocks[idx].GetName
().EndsWith(".meta"))
{
num++;
if (num % 3 == 0)
{
//
// remove .meta file
//
System.Console.Out.WriteLine("Deliberately removing file " + blocks[idx].GetName(
));
NUnit.Framework.Assert.IsTrue("Cannot remove file.", blocks[idx].Delete());
}
else
{
if (num % 3 == 1)
{
//
// shorten .meta file
//
RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
FileChannel channel = file.GetChannel();
int newsize = random.Next((int)channel.Size() / 2);
System.Console.Out.WriteLine("Deliberately truncating file " + blocks[idx].GetName
() + " to size " + newsize + " bytes.");
channel.Truncate(newsize);
file.Close();
}
else
{
//
// corrupt a few bytes of the metafile
//
RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
FileChannel channel = file.GetChannel();
long position = 0;
//
// The very first time, corrupt the meta header at offset 0
//
if (num != 2)
{
position = (long)random.Next((int)channel.Size());
}
int length = random.Next((int)(channel.Size() - position + 1));
byte[] buffer = new byte[length];
random.NextBytes(buffer);
channel.Write(ByteBuffer.Wrap(buffer), position);
System.Console.Out.WriteLine("Deliberately corrupting file " + blocks[idx].GetName
() + " at offset " + position + " length " + length);
file.Close();
}
}
}
}
//
// Now deliberately corrupt all meta blocks from the second
// directory of the first datanode
//
storageDir = cluster.GetInstanceStorageDir(0, 1);
data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid);
NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists());
blocks = data_dir.ListFiles();
NUnit.Framework.Assert.IsTrue("Blocks do not exist in data-dir", (blocks != null) &&
(blocks.Length > 0));
int count = 0;
FilePath previous = null;
for (int idx_1 = 0; idx_1 < blocks.Length; idx_1++)
{
if (blocks[idx_1].GetName().StartsWith("blk_") && blocks[idx_1].GetName().EndsWith
(".meta"))
{
//
// Move the previous metafile into the current one.
//
count++;
if (count % 2 == 0)
{
System.Console.Out.WriteLine("Deliberately insertimg bad crc into files " + blocks
[idx_1].GetName() + " " + previous.GetName());
NUnit.Framework.Assert.IsTrue("Cannot remove file.", blocks[idx_1].Delete());
NUnit.Framework.Assert.IsTrue("Cannot corrupt meta file.", previous.RenameTo(blocks
[idx_1]));
NUnit.Framework.Assert.IsTrue("Cannot recreate empty meta file.", previous.CreateNewFile
());
previous = null;
}
else
{
previous = blocks[idx_1];
}
}
}
//
// Only one replica is possibly corrupted. The other replica should still
// be good. Verify.
//
NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly.", util.CheckFiles
(fs, "/srcdat"));
System.Console.Out.WriteLine("All File still have a valid replica");
//
// set replication factor back to 1. This causes only one replica of
// of each block to remain in HDFS. The check is to make sure that
// the corrupted replica generated above is the one that gets deleted.
// This test is currently disabled until HADOOP-1557 is solved.
//
util.SetReplication(fs, "/srcdat", (short)1);
//util.waitReplication(fs, "/srcdat", (short)1);
//System.out.println("All Files done with removing replicas");
//assertTrue("Excess replicas deleted. Corrupted replicas found.",
// util.checkFiles(fs, "/srcdat"));
System.Console.Out.WriteLine("The excess-corrupted-replica test is disabled " + " pending HADOOP-1557"
);
util.Cleanup(fs, "/srcdat");
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Check that listCorruptFileBlocks works while the namenode is still in safemode.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestListCorruptFileBlocksInSafeMode()
{
MiniDFSCluster cluster = null;
Random random = new Random();
try
{
Configuration conf = new HdfsConfiguration();
// datanode scans directories
conf.SetInt(DFSConfigKeys.DfsDatanodeDirectoryscanIntervalKey, 1);
// datanode sends block reports
conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 3 * 1000);
// never leave safemode automatically
conf.SetFloat(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, 1.5f);
// start populating repl queues immediately
conf.SetFloat(DFSConfigKeys.DfsNamenodeReplQueueThresholdPctKey, 0f);
// Set short retry timeouts so this test runs faster
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
cluster = new MiniDFSCluster.Builder(conf).WaitSafeMode(false).Build();
cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave,
false);
FileSystem fs = cluster.GetFileSystem();
// create two files with one block each
DFSTestUtil util = new DFSTestUtil.Builder().SetName("testListCorruptFileBlocksInSafeMode"
).SetNumFiles(2).SetMaxLevels(1).SetMaxSize(512).Build();
util.CreateFiles(fs, "/srcdat10");
// fetch bad file list from namenode. There should be none.
ICollection <FSNamesystem.CorruptFileBlockInfo> badFiles = cluster.GetNameNode().GetNamesystem
().ListCorruptFileBlocks("/", null);
NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " corrupt files. Expecting None."
, badFiles.Count == 0);
// Now deliberately corrupt one block
FilePath storageDir = cluster.GetInstanceStorageDir(0, 0);
FilePath data_dir = MiniDFSCluster.GetFinalizedDir(storageDir, cluster.GetNamesystem
().GetBlockPoolId());
NUnit.Framework.Assert.IsTrue("data directory does not exist", data_dir.Exists());
IList <FilePath> metaFiles = MiniDFSCluster.GetAllBlockMetadataFiles(data_dir);
NUnit.Framework.Assert.IsTrue("Data directory does not contain any blocks or there was an "
+ "IO error", metaFiles != null && !metaFiles.IsEmpty());
FilePath metaFile = metaFiles[0];
RandomAccessFile file = new RandomAccessFile(metaFile, "rw");
FileChannel channel = file.GetChannel();
long position = channel.Size() - 2;
int length = 2;
byte[] buffer = new byte[length];
random.NextBytes(buffer);
channel.Write(ByteBuffer.Wrap(buffer), position);
file.Close();
Log.Info("Deliberately corrupting file " + metaFile.GetName() + " at offset " + position
+ " length " + length);
// read all files to trigger detection of corrupted replica
try
{
util.CheckFiles(fs, "/srcdat10");
}
catch (BlockMissingException)
{
System.Console.Out.WriteLine("Received BlockMissingException as expected.");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly. " + "Expecting BlockMissingException "
+ " but received IOException " + e, false);
}
// fetch bad file list from namenode. There should be one file.
badFiles = cluster.GetNameNode().GetNamesystem().ListCorruptFileBlocks("/", null);
Log.Info("Namenode has bad files. " + badFiles.Count);
NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting 1."
, badFiles.Count == 1);
// restart namenode
cluster.RestartNameNode(0);
fs = cluster.GetFileSystem();
// wait until replication queues have been initialized
while (!cluster.GetNameNode().namesystem.IsPopulatingReplQueues())
{
try
{
Log.Info("waiting for replication queues");
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
}
// read all files to trigger detection of corrupted replica
try
{
util.CheckFiles(fs, "/srcdat10");
}
catch (BlockMissingException)
{
System.Console.Out.WriteLine("Received BlockMissingException as expected.");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue("Corrupted replicas not handled properly. " + "Expecting BlockMissingException "
+ " but received IOException " + e, false);
}
// fetch bad file list from namenode. There should be one file.
badFiles = cluster.GetNameNode().GetNamesystem().ListCorruptFileBlocks("/", null);
Log.Info("Namenode has bad files. " + badFiles.Count);
NUnit.Framework.Assert.IsTrue("Namenode has " + badFiles.Count + " bad files. Expecting 1."
, badFiles.Count == 1);
// check that we are still in safe mode
NUnit.Framework.Assert.IsTrue("Namenode is not in safe mode", cluster.GetNameNode
().IsInSafeMode());
// now leave safe mode so that we can clean up
cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave,
false);
util.Cleanup(fs, "/srcdat10");
}
catch (Exception e)
{
Log.Error(StringUtils.StringifyException(e));
throw;
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public static void LoadDict(Context context)
{
try
{
bool resaveEntries = false;
dictParts = new List <byte[]>();
dictIndexes = new List <int>();
File dictFd = new File(context.FilesDir, "dict.db");
if (!dictFd.Exists())
{ // || dictFd.length() != 4961308) {
System.Console.WriteLine("DOES NOT EXIST!!!!!");
CopyFile(context, "dict.db");
dictFd = new File(context.FilesDir, "dict.db");
resaveEntries = true;
}
dictFile = new RandomAccessFile(dictFd, "r");
File idxFd = new File(context.FilesDir, "idx.db");
if (!idxFd.Exists())
{ // || idxFd.length() != 3145553) {
CopyFile(context, "idx.db");
idxFd = new File(context.FilesDir, "idx.db");
resaveEntries = true;
}
FileInputStream idxBuf = new FileInputStream(idxFd);
if (!new File(context.FilesDir, "entries.bin").Exists() || !new File(context.FilesDir, "parts.bin").Exists())
{
resaveEntries = true;
}
entries = IntBuffer.Allocate(1649830);
int index = 0;
//System.Console.WriteLine("LoadDict STEP 1");
if (idxBuf != null)
{
int readLen, offset = 0, partLen = 200000;
byte[] dictPart = new byte[partLen];
int totalRead = 0;
int totalLen = (int)idxFd.Length();
while (totalRead < totalLen && (readLen = idxBuf.Read(dictPart, offset, dictPart.Length - offset)) > 0)
{
//System.Console.WriteLine("LoadDict \ntotalRead = " + totalRead + "\ntotalLen = " + totalLen + "\nreadLen = " + readLen + "\nidxBuf.Read = " + idxBuf.Read(dictPart, offset, dictPart.Length - offset));
totalRead += readLen;
int j = offset + readLen - 1;
byte[] newDictPart = null;
if (readLen == partLen - offset)
{
//System.Console.WriteLine("LoadDict STEP 4.1 " + dictPart[j] + " :: j => " + j);
while (dictPart[j] > 0)
{
//System.Console.WriteLine("j = " + j + "\ndictPart[j] = " + dictPart[j]);
j--;
}
//System.Console.WriteLine("LoadDict STEP 4.2");
while (dictPart[j] < 0)
{
System.Console.WriteLine("j = " + j);
j--;
}
//System.Console.WriteLine("LoadDict STEP 4.3");
offset = partLen - j - 1;
//System.Console.WriteLine("LoadDict STEP 4.4");
newDictPart = new byte[Math.Min(totalLen - totalRead + offset, partLen)];
//System.Console.WriteLine("LoadDict STEP 4.5");
Java.Lang.JavaSystem.Arraycopy(dictPart, j + 1, newDictPart, 0, offset);
//Array.Copy(dictPart, j + 1, newDictPart, 0, offset);
}
else
{
offset = 0;
}
//System.Console.WriteLine("LoadDict STEP 5");
if (resaveEntries)
{
dictIndexes.Add(index);
//System.Console.WriteLine("LoadDict STEP 6");
int i = 0;
while (i <= j)
{
entries.Put(index++, i);
while (i <= j && dictPart[i] < 0)
{
i++;
}
while (i <= j && dictPart[i] >= 0)
{
i++;
}
}
}
//System.Console.WriteLine("LoadDict STEP 7");
dictParts.Add(dictPart);
dictPart = newDictPart;
//System.Console.WriteLine("LoadDict STEP 8");
}
idxBuf.Close();
}
if (resaveEntries)
{
//System.Console.WriteLine("LoadDict STEP 9");
DataOutputStream entriesOut = null, partsOut = null;
//System.Console.WriteLine("LoadDict STEP 10");
entriesOut = new DataOutputStream(context.OpenFileOutput("entries.bin", FileCreationMode.Private));
int count = entries.Capacity();
for (int i = 0; i < count; i++)
{
entriesOut.WriteInt(entries.Get(i));
}
//System.Console.WriteLine("LoadDict STEP 11");
partsOut = new DataOutputStream(context.OpenFileOutput("parts.bin", FileCreationMode.Private));
foreach (int i in dictIndexes)
{
partsOut.WriteInt(i);
}
//System.Console.WriteLine("LoadDict STEP 12");
if (entriesOut != null)
{
entriesOut.Flush();
entriesOut.Close();
}
if (partsOut != null)
{
partsOut.Flush();
partsOut.Close();
}
}
else
{
//System.Console.WriteLine("LoadDict NOW RESAVING ENTRIES");
string documentpath = System.Environment.GetFolderPath(System.Environment.SpecialFolder.Personal);
Java.IO.File sdpath = global::Android.OS.Environment.GetExternalStoragePublicDirectory(global::Android.OS.Environment.DirectoryDownloads);
File entriesDB = new File(documentpath, "entries.bin");
File partsDB = new File(documentpath, "parts.bin");
FileInputStream entriesIn = null, partsIn = null;
//entriesIn = context.OpenFileInput("entries.bin");
entriesIn = new FileInputStream(entriesDB);
//entriesIn = new FileInputStream(new File("entries.bin"));
FileChannel file = entriesIn.Channel;
ByteBuffer bb = ByteBuffer.Allocate(4 * 1649830);
file.Read(bb);
bb.Rewind();
entries = bb.AsIntBuffer();
file.Close();
partsIn = new FileInputStream(partsDB);
//partsIn = new FileInputStream(new File("parts.bin"));
//partsIn = (context.OpenFileInput("parts.bin");
file = partsIn.Channel;
bb = ByteBuffer.Allocate((int)file.Size());
file.Read(bb);
bb.Rewind();
IntBuffer ib = bb.AsIntBuffer();
int count = ib.Capacity();
//System.Console.WriteLine("LoadDict STEP 99 " + count);
for (int i = 0; i < count; i++)
{
dictIndexes.Add(ib.Get(i));
}
file.Close();
if (entriesIn != null)
{
entriesIn.Close();
}
if (partsIn != null)
{
partsIn.Close();
}
}
}
catch (Exception e)
{
System.Console.WriteLine("Dict LoadDict ERROR => " + e.Message);
Log.Equals("chinesreader", e.Message);
}
byteBuffer = new byte[1090];
sharedPrefs = PreferenceManager.GetDefaultSharedPreferences(context);
}