/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
public ChecksumFSInputChecker(ChecksumFs fs, Path file, int bufferSize)
: base(file, fs.GetFileStatus(file).GetReplication())
{
this.datas = fs.GetRawFs().Open(file, bufferSize);
this.fs = fs;
Path sumFile = fs.GetChecksumFile(file);
try
{
int sumBufferSize = fs.GetSumBufferSize(fs.GetBytesPerSum(), bufferSize);
sums = fs.GetRawFs().Open(sumFile, sumBufferSize);
byte[] version = new byte[ChecksumVersion.Length];
sums.ReadFully(version);
if (!Arrays.Equals(version, ChecksumVersion))
{
throw new IOException("Not a checksum file: " + sumFile);
}
this.bytesPerSum = sums.ReadInt();
Set(fs.verifyChecksum, DataChecksum.NewCrc32(), bytesPerSum, 4);
}
catch (FileNotFoundException)
{
// quietly ignore
Set(fs.verifyChecksum, null, 1, 0);
}
catch (IOException e)
{
// loudly ignore
Log.Warn("Problem opening checksum file: " + file + ". Ignoring exception: ", e);
Set(fs.verifyChecksum, null, 1, 0);
}
}
public virtual void TestOpenManyFilesViaTcp()
{
int NumOpens = 500;
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false);
MiniDFSCluster cluster = null;
FSDataInputStream[] streams = new FSDataInputStream[NumOpens];
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem dfs = cluster.GetFileSystem();
Path TestPath = new Path("/testFile");
DFSTestUtil.CreateFile(dfs, TestPath, 131072, (short)1, 1);
for (int i = 0; i < NumOpens; i++)
{
streams[i] = dfs.Open(TestPath);
Log.Info("opening file " + i + "...");
NUnit.Framework.Assert.IsTrue(-1 != streams[i].Read());
streams[i].Unbuffer();
}
}
finally
{
foreach (FSDataInputStream stream in streams)
{
IOUtils.Cleanup(null, stream);
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Construct given a
/// <see cref="FileContext"/>
/// and a
/// <see cref="Path"/>
/// .
/// </summary>
/// <exception cref="System.IO.IOException"/>
public AvroFSInput(FileContext fc, Path p)
{
FileStatus status = fc.GetFileStatus(p);
this.len = status.GetLen();
this.stream = fc.Open(p);
}
public virtual void TestStatistics()
{
URI fsUri = GetFsUri();
FileSystem.Statistics stats = FileContext.GetStatistics(fsUri);
Assert.Equal(0, stats.GetBytesRead());
Path filePath = fileContextTestHelper.GetTestRootPath(fc, "file1");
FileContextTestHelper.CreateFile(fc, filePath, numBlocks, blockSize);
Assert.Equal(0, stats.GetBytesRead());
VerifyWrittenBytes(stats);
FSDataInputStream fstr = fc.Open(filePath);
byte[] buf = new byte[blockSize];
int bytesRead = fstr.Read(buf, 0, blockSize);
fstr.Read(0, buf, 0, blockSize);
Assert.Equal(blockSize, bytesRead);
VerifyReadBytes(stats);
VerifyWrittenBytes(stats);
VerifyReadBytes(FileContext.GetStatistics(GetFsUri()));
IDictionary <URI, FileSystem.Statistics> statsMap = FileContext.GetAllStatistics();
URI exactUri = GetSchemeAuthorityUri();
VerifyWrittenBytes(statsMap[exactUri]);
fc.Delete(filePath, true);
}
public virtual void TestBufferedFSInputStream()
{
Configuration conf = new Configuration();
conf.SetClass("fs.file.impl", typeof(RawLocalFileSystem), typeof(FileSystem));
conf.SetInt(CommonConfigurationKeysPublic.IoFileBufferSizeKey, 4096);
FileSystem fs = FileSystem.NewInstance(conf);
byte[] buf = new byte[10 * 1024];
new Random().NextBytes(buf);
// Write random bytes to file
FSDataOutputStream stream = fs.Create(TestPath);
try
{
stream.Write(buf);
}
finally
{
stream.Close();
}
Random r = new Random();
FSDataInputStream stm = fs.Open(TestPath);
// Record the sequence of seeks and reads which trigger a failure.
int[] seeks = new int[10];
int[] reads = new int[10];
try
{
for (int i = 0; i < 1000; i++)
{
int seekOff = r.Next(buf.Length);
int toRead = r.Next(Math.Min(buf.Length - seekOff, 32000));
seeks[i % seeks.Length] = seekOff;
reads[i % reads.Length] = toRead;
VerifyRead(stm, buf, seekOff, toRead);
}
}
catch (Exception afe)
{
StringBuilder sb = new StringBuilder();
sb.Append("Sequence of actions:\n");
for (int j = 0; j < seeks.Length; j++)
{
sb.Append("seek @ ").Append(seeks[j]).Append(" ").Append("read ").Append(reads[j
]).Append("\n");
}
System.Console.Error.WriteLine(sb.ToString());
throw;
}
finally
{
stm.Close();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestInputStreamClosedTwice()
{
//HADOOP-4760 according to Closeable#close() closing already-closed
//streams should have no effect.
Org.Apache.Hadoop.FS.Path src = Path("/test/hadoop/file");
CreateFile(src);
FSDataInputStream @in = fs.Open(src);
@in.Close();
@in.Close();
}
public virtual void TestFallbackRead()
{
HdfsConfiguration conf = InitZeroCopyTest();
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
int TestFileLength = 16385;
int RandomSeed = 23453;
FSDataInputStream fsIn = null;
DistributedFileSystem fs = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
byte[] original = new byte[TestFileLength];
IOUtils.ReadFully(fsIn, original, 0, TestFileLength);
fsIn.Close();
fsIn = fs.Open(TestPath);
TestFallbackImpl(fsIn, original);
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
private string ReadFile(string @out)
{
Path path = new Path(@out);
FileStatus stat = lfs.GetFileStatus(path);
FSDataInputStream @in = lfs.Open(path);
byte[] buffer = new byte[(int)stat.GetLen()];
@in.ReadFully(buffer);
@in.Close();
lfs.Delete(path, false);
return(Runtime.GetStringForBytes(buffer));
}
/// <exception cref="System.IO.IOException"/>
private void VerifyRead(FSDataInputStream stm, byte[] fileContents, int seekOff,
int toRead)
{
byte[] @out = new byte[toRead];
stm.Seek(seekOff);
stm.ReadFully(@out);
byte[] expected = Arrays.CopyOfRange(fileContents, seekOff, seekOff + toRead);
if (!Arrays.Equals(@out, expected))
{
string s = "\nExpected: " + StringUtils.ByteToHexString(expected) + "\ngot: "
+ StringUtils.ByteToHexString(@out) + "\noff=" + seekOff + " len=" + toRead;
NUnit.Framework.Assert.Fail(s);
}
}
public virtual void TestUnbufferClosesSockets()
{
Configuration conf = new Configuration();
// Set a new ClientContext. This way, we will have our own PeerCache,
// rather than sharing one with other unit tests.
conf.Set(DFSConfigKeys.DfsClientContext, "testUnbufferClosesSocketsContext");
// Disable short-circuit reads. With short-circuit, we wouldn't hold open a
// TCP socket.
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false);
// Set a really long socket timeout to avoid test timing issues.
conf.SetLong(DFSConfigKeys.DfsClientSocketTimeoutKey, 100000000L);
conf.SetLong(DFSConfigKeys.DfsClientSocketCacheExpiryMsecKey, 100000000L);
MiniDFSCluster cluster = null;
FSDataInputStream stream = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.NewInstance(conf);
Path TestPath = new Path("/test1");
DFSTestUtil.CreateFile(dfs, TestPath, 128, (short)1, 1);
stream = dfs.Open(TestPath);
// Read a byte. This will trigger the creation of a block reader.
stream.Seek(2);
int b = stream.Read();
NUnit.Framework.Assert.IsTrue(-1 != b);
// The Peer cache should start off empty.
PeerCache cache = dfs.GetClient().GetClientContext().GetPeerCache();
NUnit.Framework.Assert.AreEqual(0, cache.Size());
// Unbuffer should clear the block reader and return the socket to the
// cache.
stream.Unbuffer();
stream.Seek(2);
NUnit.Framework.Assert.AreEqual(1, cache.Size());
int b2 = stream.Read();
NUnit.Framework.Assert.AreEqual(b, b2);
}
finally
{
if (stream != null)
{
IOUtils.Cleanup(null, stream);
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Map(Text key, LongWritable value, OutputCollector <K, LongWritable
> collector, Reporter reporter)
{
string name = key.ToString();
long size = value.Get();
long seed = long.Parse(name);
if (size == 0)
{
return;
}
reporter.SetStatus("opening " + name);
FSDataInputStream @in = fs.Open(new Path(DataDir, name));
try
{
for (int i = 0; i < SeeksPerFile; i++)
{
// generate a random position
long position = Math.Abs(random.NextLong()) % size;
// seek file to that position
reporter.SetStatus("seeking " + name);
@in.Seek(position);
byte b = @in.ReadByte();
// check that byte matches
byte checkByte = 0;
// advance random state to that position
random.SetSeed(seed);
for (int p = 0; p <= position; p += check.Length)
{
reporter.SetStatus("generating data for " + name);
if (fastCheck)
{
checkByte = unchecked ((byte)random.Next(byte.MaxValue));
}
else
{
random.NextBytes(check);
checkByte = check[(int)(position % check.Length)];
}
}
NUnit.Framework.Assert.AreEqual(b, checkByte);
}
}
finally
{
@in.Close();
}
}
public virtual void TestStreamType()
{
Path testPath = new Path(TestRootDir, "testStreamType");
localFs.Create(testPath).Close();
FSDataInputStream @in = null;
localFs.SetVerifyChecksum(true);
@in = localFs.Open(testPath);
Assert.True("stream is input checker", @in.GetWrappedStream() is
FSInputChecker);
localFs.SetVerifyChecksum(false);
@in = localFs.Open(testPath);
NUnit.Framework.Assert.IsFalse("stream is not input checker", @in.GetWrappedStream
() is FSInputChecker);
}
/// <exception cref="System.IO.IOException"/>
public override IDisposable GetIOStream(string name)
{
// IOMapperBase
Path filePath = new Path(GetDataDir(GetConf()), name);
this.fileSize = fs.GetFileStatus(filePath).GetLen();
InputStream @in = fs.Open(filePath);
if (compressionCodec != null)
{
@in = new FSDataInputStream(compressionCodec.CreateInputStream(@in));
}
Log.Info("in = " + @in.GetType().FullName);
Log.Info("skipSize = " + skipSize);
return(@in);
}
/// <exception cref="System.IO.IOException"/>
internal static string ReadFile(FileSystem fs, Path name, int buflen)
{
byte[] b = new byte[buflen];
int offset = 0;
FSDataInputStream @in = fs.Open(name);
for (int remaining; (remaining = b.Length - offset) > 0 && (n = @in.Read(b, offset
, remaining)) != -1; offset += n)
{
}
Assert.Equal(offset, Math.Min(b.Length, @in.GetPos()));
@in.Close();
string s = Runtime.GetStringForBytes(b, 0, offset);
return(s);
}
/// <exception cref="System.IO.IOException"/>
private void VerifyFile(FileSystem fs, Path file, int bytesToVerify, byte[] expectedBytes
)
{
FSDataInputStream @in = fs.Open(file);
try
{
byte[] readBuf = new byte[bytesToVerify];
@in.ReadFully(readBuf, 0, bytesToVerify);
for (int i = 0; i < bytesToVerify; i++)
{
Assert.Equal(expectedBytes[i], readBuf[i]);
}
}
finally
{
@in.Close();
}
}
/// <exception cref="System.IO.IOException"/>
internal override object DoIO(Reporter reporter, string name, long offset)
{
// open file
FSDataInputStream @in = null;
Path p = new Path(name);
try
{
@in = fs.Open(p);
}
catch (IOException)
{
return(name + "@(missing)");
}
@in.Seek(offset);
long actualSize = 0;
try
{
long blockSize = fs.GetDefaultBlockSize(p);
reporter.SetStatus("reading " + name + "@" + offset + "/" + blockSize);
for (int curSize = bufferSize; curSize == bufferSize && actualSize < blockSize; actualSize
+= curSize)
{
curSize = @in.Read(buffer, 0, bufferSize);
}
}
catch (IOException)
{
Log.Info("Corrupted block detected in \"" + name + "\" at " + offset);
return(name + "@" + offset);
}
finally
{
@in.Close();
}
return(actualSize);
}
public virtual void TestTruncatedChecksum()
{
Path testPath = new Path(TestRootDir, "testtruncatedcrc");
FSDataOutputStream fout = localFs.Create(testPath);
fout.Write(Runtime.GetBytesForString("testing truncation"));
fout.Close();
// Read in the checksum
Path checksumFile = localFs.GetChecksumFile(testPath);
FileSystem rawFs = localFs.GetRawFileSystem();
FSDataInputStream checksumStream = rawFs.Open(checksumFile);
byte[] buf = new byte[8192];
int read = checksumStream.Read(buf, 0, buf.Length);
checksumStream.Close();
// Now rewrite the checksum file with the last byte missing
FSDataOutputStream replaceStream = rawFs.Create(checksumFile);
replaceStream.Write(buf, 0, read - 1);
replaceStream.Close();
// Now reading the file should fail with a ChecksumException
try
{
FileSystemTestHelper.ReadFile(localFs, testPath, 1024);
NUnit.Framework.Assert.Fail("Did not throw a ChecksumException when reading truncated "
+ "crc file");
}
catch (ChecksumException)
{
}
// telling it not to verify checksums, should avoid issue.
localFs.SetVerifyChecksum(false);
string str = FileSystemTestHelper.ReadFile(localFs, testPath, 1024).ToString();
Assert.True("read", "testing truncation".Equals(str));
}
/// <summary>Write a file and read it in, validating the result.</summary>
/// <remarks>
/// Write a file and read it in, validating the result. Optional flags control
/// whether file overwrite operations should be enabled, and whether the
/// file should be deleted afterwards.
/// If there is a mismatch between what was written and what was expected,
/// a small range of bytes either side of the first error are logged to aid
/// diagnosing what problem occurred -whether it was a previous file
/// or a corrupting of the current file. This assumes that two
/// sequential runs to the same path use datasets with different character
/// moduli.
/// </remarks>
/// <param name="path">path to write to</param>
/// <param name="len">length of data</param>
/// <param name="overwrite">should the create option allow overwrites?</param>
/// <param name="delete">
/// should the file be deleted afterwards? -with a verification
/// that it worked. Deletion is not attempted if an assertion has failed
/// earlier -it is not in a <code>finally{}</code> block.
/// </param>
/// <exception cref="System.IO.IOException">IO problems</exception>
protected internal virtual void WriteAndRead(Org.Apache.Hadoop.FS.Path path, byte
[] src, int len, bool overwrite, bool delete)
{
Assert.True("Not enough data in source array to write " + len +
" bytes", src.Length >= len);
fs.Mkdirs(path.GetParent());
FSDataOutputStream @out = fs.Create(path, overwrite, fs.GetConf().GetInt("io.file.buffer.size"
, 4096), (short)1, GetBlockSize());
@out.Write(src, 0, len);
@out.Close();
Assert.True("Exists", fs.Exists(path));
Assert.Equal("Length", len, fs.GetFileStatus(path).GetLen());
FSDataInputStream @in = fs.Open(path);
byte[] buf = new byte[len];
@in.ReadFully(0, buf);
@in.Close();
Assert.Equal(len, buf.Length);
int errors = 0;
int first_error_byte = -1;
for (int i = 0; i < len; i++)
{
if (src[i] != buf[i])
{
if (errors == 0)
{
first_error_byte = i;
}
errors++;
}
}
if (errors > 0)
{
string message = string.Format(" %d errors in file of length %d", errors, len);
Log.Warn(message);
// the range either side of the first error to print
// this is a purely arbitrary number, to aid user debugging
int overlap = 10;
for (int i_1 = Math.Max(0, first_error_byte - overlap); i_1 < Math.Min(first_error_byte
+ overlap, len); i_1++)
{
byte actual = buf[i_1];
byte expected = src[i_1];
string letter = ToChar(actual);
string line = string.Format("[%04d] %2x %s\n", i_1, actual, letter);
if (expected != actual)
{
line = string.Format("[%04d] %2x %s -expected %2x %s\n", i_1, actual, letter, expected
, ToChar(expected));
}
Log.Warn(line);
}
Fail(message);
}
if (delete)
{
bool deleted = fs.Delete(path, false);
Assert.True("Deleted", deleted);
NUnit.Framework.Assert.IsFalse("No longer exists", fs.Exists(path));
}
}
/// <summary>
/// Test that we can zero-copy read cached data even without disabling
/// checksums.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestZeroCopyReadOfCachedData()
{
BlockReaderTestUtil.EnableShortCircuitShmTracing();
BlockReaderTestUtil.EnableBlockReaderFactoryTracing();
BlockReaderTestUtil.EnableHdfsCachingTracing();
int TestFileLength = BlockSize;
Path TestPath = new Path("/a");
int RandomSeed = 23453;
HdfsConfiguration conf = InitZeroCopyTest();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
string Context = "testZeroCopyReadOfCachedData";
conf.Set(DFSConfigKeys.DfsClientContext, Context);
conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, DFSTestUtil.RoundUpToMultiple
(TestFileLength, (int)NativeIO.POSIX.GetCacheManipulator().GetOperatingSystemPageSize
()));
MiniDFSCluster cluster = null;
ByteBuffer result = null;
ByteBuffer result2 = null;
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
FsDatasetSpi <object> fsd = cluster.GetDataNodes()[0].GetFSDataset();
DistributedFileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed);
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
byte[] original = DFSTestUtil.CalculateFileContentsFromSeed(RandomSeed, TestFileLength
);
// Prior to caching, the file can't be read via zero-copy
FSDataInputStream fsIn = fs.Open(TestPath);
try
{
result = fsIn.Read(null, TestFileLength / 2, EnumSet.NoneOf <ReadOption>());
NUnit.Framework.Assert.Fail("expected UnsupportedOperationException");
}
catch (NotSupportedException)
{
}
// expected
// Cache the file
fs.AddCachePool(new CachePoolInfo("pool1"));
long directiveId = fs.AddCacheDirective(new CacheDirectiveInfo.Builder().SetPath(
TestPath).SetReplication((short)1).SetPool("pool1").Build());
int numBlocks = (int)Math.Ceil((double)TestFileLength / BlockSize);
DFSTestUtil.VerifyExpectedCacheUsage(DFSTestUtil.RoundUpToMultiple(TestFileLength
, BlockSize), numBlocks, cluster.GetDataNodes()[0].GetFSDataset());
try
{
result = fsIn.Read(null, TestFileLength, EnumSet.NoneOf <ReadOption>());
}
catch (NotSupportedException)
{
NUnit.Framework.Assert.Fail("expected to be able to read cached file via zero-copy"
);
}
Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray
(result));
// Test that files opened after the cache operation has finished
// still get the benefits of zero-copy (regression test for HDFS-6086)
FSDataInputStream fsIn2 = fs.Open(TestPath);
try
{
result2 = fsIn2.Read(null, TestFileLength, EnumSet.NoneOf <ReadOption>());
}
catch (NotSupportedException)
{
NUnit.Framework.Assert.Fail("expected to be able to read cached file via zero-copy"
);
}
Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray
(result2));
fsIn2.ReleaseBuffer(result2);
fsIn2.Close();
// check that the replica is anchored
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fs, TestPath);
ShortCircuitCache cache = ClientContext.Get(Context, new DFSClient.Conf(conf)).GetShortCircuitCache
();
WaitForReplicaAnchorStatus(cache, firstBlock, true, true, 1);
// Uncache the replica
fs.RemoveCacheDirective(directiveId);
WaitForReplicaAnchorStatus(cache, firstBlock, false, true, 1);
fsIn.ReleaseBuffer(result);
WaitForReplicaAnchorStatus(cache, firstBlock, false, false, 1);
DFSTestUtil.VerifyExpectedCacheUsage(0, 0, fsd);
fsIn.Close();
fs.Close();
cluster.Shutdown();
}
public virtual void TestClientMmapDisable()
{
HdfsConfiguration conf = InitZeroCopyTest();
conf.SetBoolean(DFSConfigKeys.DfsClientMmapEnabled, false);
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
int TestFileLength = 16385;
int RandomSeed = 23453;
string Context = "testClientMmapDisable";
FSDataInputStream fsIn = null;
DistributedFileSystem fs = null;
conf.Set(DFSConfigKeys.DfsClientContext, Context);
try
{
// With DFS_CLIENT_MMAP_ENABLED set to false, we should not do memory
// mapped reads.
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed);
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
fsIn = fs.Open(TestPath);
try
{
fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.Fail("expected zero-copy read to fail when client mmaps "
+ "were disabled.");
}
catch (NotSupportedException)
{
}
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
fsIn = null;
fs = null;
cluster = null;
try
{
// Now try again with DFS_CLIENT_MMAP_CACHE_SIZE == 0. It should work.
conf.SetBoolean(DFSConfigKeys.DfsClientMmapEnabled, true);
conf.SetInt(DFSConfigKeys.DfsClientMmapCacheSize, 0);
conf.Set(DFSConfigKeys.DfsClientContext, Context + ".1");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed);
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
fsIn = fs.Open(TestPath);
ByteBuffer buf = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums));
fsIn.ReleaseBuffer(buf);
// Test EOF behavior
IOUtils.SkipFully(fsIn, TestFileLength - 1);
buf = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(null, buf);
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestZeroCopyReads()
{
HdfsConfiguration conf = InitZeroCopyTest();
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
FSDataInputStream fsIn = null;
int TestFileLength = 3 * BlockSize;
FileSystem fs = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, 7567L);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
byte[] original = new byte[TestFileLength];
IOUtils.ReadFully(fsIn, original, 0, TestFileLength);
fsIn.Close();
fsIn = fs.Open(TestPath);
ByteBuffer result = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums
));
NUnit.Framework.Assert.AreEqual(BlockSize, result.Remaining());
HdfsDataInputStream dfsIn = (HdfsDataInputStream)fsIn;
NUnit.Framework.Assert.AreEqual(BlockSize, dfsIn.GetReadStatistics().GetTotalBytesRead
());
NUnit.Framework.Assert.AreEqual(BlockSize, dfsIn.GetReadStatistics().GetTotalZeroCopyBytesRead
());
Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray
(result));
fsIn.ReleaseBuffer(result);
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void Test2GBMmapLimit()
{
Assume.AssumeTrue(BlockReaderTestUtil.ShouldTestLargeFiles());
HdfsConfiguration conf = InitZeroCopyTest();
long TestFileLength = 2469605888L;
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "NULL");
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, TestFileLength);
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
string Context = "test2GBMmapLimit";
conf.Set(DFSConfigKeys.DfsClientContext, Context);
FSDataInputStream fsIn = null;
FSDataInputStream fsIn2 = null;
ByteBuffer buf1 = null;
ByteBuffer buf2 = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, unchecked ((int)(0xB
)));
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
fsIn = fs.Open(TestPath);
buf1 = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(1, buf1.Remaining());
fsIn.ReleaseBuffer(buf1);
buf1 = null;
fsIn.Seek(2147483640L);
buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(7, buf1.Remaining());
NUnit.Framework.Assert.AreEqual(int.MaxValue, buf1.Limit());
fsIn.ReleaseBuffer(buf1);
buf1 = null;
NUnit.Framework.Assert.AreEqual(2147483647L, fsIn.GetPos());
try
{
buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.Fail("expected UnsupportedOperationException");
}
catch (NotSupportedException)
{
}
// expected; can't read past 2GB boundary.
fsIn.Close();
fsIn = null;
// Now create another file with normal-sized blocks, and verify we
// can read past 2GB
Path TestPath2 = new Path("/b");
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 268435456L);
DFSTestUtil.CreateFile(fs, TestPath2, 1024 * 1024, TestFileLength, 268435456L, (short
)1, unchecked ((int)(0xA)));
fsIn2 = fs.Open(TestPath2);
fsIn2.Seek(2147483640L);
buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(8, buf2.Remaining());
NUnit.Framework.Assert.AreEqual(2147483648L, fsIn2.GetPos());
fsIn2.ReleaseBuffer(buf2);
buf2 = null;
buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums));
NUnit.Framework.Assert.AreEqual(1024, buf2.Remaining());
NUnit.Framework.Assert.AreEqual(2147484672L, fsIn2.GetPos());
fsIn2.ReleaseBuffer(buf2);
buf2 = null;
}
finally
{
if (buf1 != null)
{
fsIn.ReleaseBuffer(buf1);
}
if (buf2 != null)
{
fsIn2.ReleaseBuffer(buf2);
}
IOUtils.Cleanup(null, fsIn, fsIn2);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// When mark() is used on BufferedInputStream, the request
/// size on the checksum file system can be small.
/// </summary>
/// <remarks>
/// When mark() is used on BufferedInputStream, the request
/// size on the checksum file system can be small. However,
/// checksum file system currently depends on the request size
/// >= bytesPerSum to work properly.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestTruncatedInputBug()
{
int ioBufSize = 512;
int fileSize = ioBufSize * 4;
int filePos = 0;
Configuration conf = new Configuration();
conf.SetInt("io.file.buffer.size", ioBufSize);
FileSystem fileSys = FileSystem.GetLocal(conf);
try
{
// First create a test input file.
Path testFile = new Path(TestRootDir, "HADOOP-1489");
WriteFile(fileSys, testFile, fileSize);
Assert.True(fileSys.Exists(testFile));
Assert.True(fileSys.GetFileStatus(testFile).GetLen() == fileSize
);
// Now read the file for ioBufSize bytes
FSDataInputStream @in = fileSys.Open(testFile, ioBufSize);
// seek beyond data buffered by open
filePos += ioBufSize * 2 + (ioBufSize - 10);
@in.Seek(filePos);
// read 4 more bytes before marking
for (int i = 0; i < 4; ++i)
{
if (@in.Read() == -1)
{
break;
}
++filePos;
}
// Now set mark() to trigger the bug
// NOTE: in the fixed code, mark() does nothing (not supported) and
// hence won't trigger this bug.
@in.Mark(1);
System.Console.Out.WriteLine("MARKED");
// Try to read the rest
while (filePos < fileSize)
{
if (@in.Read() == -1)
{
break;
}
++filePos;
}
@in.Close();
System.Console.Out.WriteLine("Read " + filePos + " bytes." + " file size=" + fileSize
);
Assert.True(filePos == fileSize);
}
finally
{
try
{
fileSys.Close();
}
catch (Exception)
{
}
}
}
/// <summary>
/// Construct given an
/// <see cref="FSDataInputStream"/>
/// and its length.
/// </summary>
public AvroFSInput(FSDataInputStream @in, long len)
{
this.stream = @in;
this.len = len;
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestReportChecksumFailure()
{
@base.Mkdirs();
Assert.True(@base.Exists() && @base.IsDirectory());
FilePath dir1 = new FilePath(@base, "dir1");
FilePath dir2 = new FilePath(dir1, "dir2");
dir2.Mkdirs();
Assert.True(dir2.Exists() && FileUtil.CanWrite(dir2));
string dataFileName = "corruptedData";
Path dataPath = new Path(new FilePath(dir2, dataFileName).ToURI());
Path checksumPath = fileSys.GetChecksumFile(dataPath);
FSDataOutputStream fsdos = fileSys.Create(dataPath);
try
{
fsdos.WriteUTF("foo");
}
finally
{
fsdos.Close();
}
Assert.True(fileSys.PathToFile(dataPath).Exists());
long dataFileLength = fileSys.GetFileStatus(dataPath).GetLen();
Assert.True(dataFileLength > 0);
// check the the checksum file is created and not empty:
Assert.True(fileSys.PathToFile(checksumPath).Exists());
long checksumFileLength = fileSys.GetFileStatus(checksumPath).GetLen();
Assert.True(checksumFileLength > 0);
// this is a hack to force the #reportChecksumFailure() method to stop
// climbing up at the 'base' directory and use 'dir1/bad_files' as the
// corrupted files storage:
FileUtil.SetWritable(@base, false);
FSDataInputStream dataFsdis = fileSys.Open(dataPath);
FSDataInputStream checksumFsdis = fileSys.Open(checksumPath);
bool retryIsNecessary = fileSys.ReportChecksumFailure(dataPath, dataFsdis, 0, checksumFsdis
, 0);
Assert.True(!retryIsNecessary);
// the data file should be moved:
Assert.True(!fileSys.PathToFile(dataPath).Exists());
// the checksum file should be moved:
Assert.True(!fileSys.PathToFile(checksumPath).Exists());
// check that the files exist in the new location where they were moved:
FilePath[] dir1files = dir1.ListFiles(new _FileFilter_352());
Assert.True(dir1files != null);
Assert.True(dir1files.Length == 1);
FilePath badFilesDir = dir1files[0];
FilePath[] badFiles = badFilesDir.ListFiles();
Assert.True(badFiles != null);
Assert.True(badFiles.Length == 2);
bool dataFileFound = false;
bool checksumFileFound = false;
foreach (FilePath badFile in badFiles)
{
if (badFile.GetName().StartsWith(dataFileName))
{
Assert.True(dataFileLength == badFile.Length());
dataFileFound = true;
}
else
{
if (badFile.GetName().Contains(dataFileName + ".crc"))
{
Assert.True(checksumFileLength == badFile.Length());
checksumFileFound = true;
}
}
}
Assert.True(dataFileFound);
Assert.True(checksumFileFound);
}
/// <summary>Report a checksum error to the file system.</summary>
/// <param name="f">the file name containing the error</param>
/// <param name="in">the stream open on the file</param>
/// <param name="inPos">the position of the beginning of the bad data in the file</param>
/// <param name="sums">the stream open on the checksum file</param>
/// <param name="sumsPos">
/// the position of the beginning of the bad data in the
/// checksum file
/// </param>
/// <returns>if retry is neccessary</returns>
public virtual bool ReportChecksumFailure(Path f, FSDataInputStream @in, long inPos
, FSDataInputStream sums, long sumsPos)
{
return(false);
}
public virtual void TestZeroCopyMmapCache()
{
HdfsConfiguration conf = InitZeroCopyTest();
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
int TestFileLength = 5 * BlockSize;
int RandomSeed = 23453;
string Context = "testZeroCopyMmapCacheContext";
FSDataInputStream fsIn = null;
ByteBuffer[] results = new ByteBuffer[] { null, null, null, null };
DistributedFileSystem fs = null;
conf.Set(DFSConfigKeys.DfsClientContext, Context);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
byte[] original = new byte[TestFileLength];
IOUtils.ReadFully(fsIn, original, 0, TestFileLength);
fsIn.Close();
fsIn = fs.Open(TestPath);
ShortCircuitCache cache = ClientContext.Get(Context, new DFSClient.Conf(conf)).GetShortCircuitCache
();
cache.Accept(new TestEnhancedByteBufferAccess.CountingVisitor(0, 5, 5, 0));
results[0] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums));
fsIn.Seek(0);
results[1] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums));
// The mmap should be of the first block of the file.
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fs, TestPath);
cache.Accept(new _CacheVisitor_373(firstBlock));
// The replica should not yet be evictable, since we have it open.
// Read more blocks.
results[2] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums));
results[3] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums));
// we should have 3 mmaps, 1 evictable
cache.Accept(new TestEnhancedByteBufferAccess.CountingVisitor(3, 5, 2, 0));
// After we close the cursors, the mmaps should be evictable for
// a brief period of time. Then, they should be closed (we're
// using a very quick timeout)
foreach (ByteBuffer buffer in results)
{
if (buffer != null)
{
fsIn.ReleaseBuffer(buffer);
}
}
fsIn.Close();
GenericTestUtils.WaitFor(new _Supplier_407(cache), 10, 60000);
cache.Accept(new TestEnhancedByteBufferAccess.CountingVisitor(0, -1, -1, -1));
fs.Close();
cluster.Shutdown();
}