/// <summary>
/// Test to run benchmarks between short circuit read vs regular read with
/// specified number of threads simultaneously reading.
/// </summary>
/// <remarks>
/// Test to run benchmarks between short circuit read vs regular read with
/// specified number of threads simultaneously reading.
/// <br />
/// Run this using the following command:
/// bin/hadoop --config confdir \
/// org.apache.hadoop.hdfs.TestShortCircuitLocalRead \
/// <shortcircuit on?> <checsum on?> <Number of threads>
/// </remarks>
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
if (args.Length != 3)
{
System.Console.Out.WriteLine("Usage: test shortcircuit checksum threadCount");
System.Environment.Exit(1);
}
bool shortcircuit = Sharpen.Extensions.ValueOf(args[0]);
bool checksum = Sharpen.Extensions.ValueOf(args[1]);
int threadCount = System.Convert.ToInt32(args[2]);
// Setup create a file
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, shortcircuit);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "/tmp/TestShortCircuitLocalRead._PORT"
);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, checksum);
//Override fileSize and DATA_TO_WRITE to much larger values for benchmark test
int fileSize = 1000 * blockSize + 100;
// File with 1000 blocks
byte[] dataToWrite = AppendTestUtil.RandomBytes(seed, fileSize);
// create a new file in home directory. Do not close it.
Path file1 = new Path("filelocal.dat");
FileSystem fs = FileSystem.Get(conf);
FSDataOutputStream stm = CreateFile(fs, file1, 1);
stm.Write(dataToWrite);
stm.Close();
long start = Time.Now();
int iteration = 20;
Sharpen.Thread[] threads = new Sharpen.Thread[threadCount];
for (int i = 0; i < threadCount; i++)
{
threads[i] = new _Thread_554(iteration, fs, file1, dataToWrite, conf);
}
for (int i_1 = 0; i_1 < threadCount; i_1++)
{
threads[i_1].Start();
}
for (int i_2 = 0; i_2 < threadCount; i_2++)
{
threads[i_2].Join();
}
long end = Time.Now();
System.Console.Out.WriteLine("Iteration " + iteration + " took " + (end - start));
fs.Delete(file1, false);
}
/// <summary>
/// Test that file data can be read by reading the block file
/// directly from the local store.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void DoTestShortCircuitReadImpl(bool ignoreChecksum, int size, int
readOffset, string shortCircuitUser, string readingUser, bool legacyShortCircuitFails
)
{
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, ignoreChecksum
);
// Set a random client context name so that we don't share a cache with
// other invocations of this function.
conf.Set(DFSConfigKeys.DfsClientContext, UUID.RandomUUID().ToString());
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, new FilePath(sockDir.GetDir(), "TestShortCircuitLocalRead._PORT.sock"
).GetAbsolutePath());
if (shortCircuitUser != null)
{
conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, shortCircuitUser);
conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true);
}
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(
true).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// check that / exists
Path path = new Path("/");
NUnit.Framework.Assert.IsTrue("/ should be a directory", fs.GetFileStatus(path).IsDirectory
() == true);
byte[] fileData = AppendTestUtil.RandomBytes(seed, size);
Path file1 = fs.MakeQualified(new Path("filelocal.dat"));
FSDataOutputStream stm = CreateFile(fs, file1, 1);
stm.Write(fileData);
stm.Close();
URI uri = cluster.GetURI();
CheckFileContent(uri, file1, fileData, readOffset, readingUser, conf, legacyShortCircuitFails
);
CheckFileContentDirect(uri, file1, fileData, readOffset, readingUser, conf, legacyShortCircuitFails
);
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
/// <summary>
/// Test that file data can be read by reading the block
/// through RemoteBlockReader
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void DoTestShortCircuitReadWithRemoteBlockReader(bool ignoreChecksum
, int size, string shortCircuitUser, int readOffset, bool shortCircuitFails)
{
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreader, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(
true).Build();
FileSystem fs = cluster.GetFileSystem();
// check that / exists
Path path = new Path("/");
URI uri = cluster.GetURI();
NUnit.Framework.Assert.IsTrue("/ should be a directory", fs.GetFileStatus(path).IsDirectory
() == true);
byte[] fileData = AppendTestUtil.RandomBytes(seed, size);
Path file1 = new Path("filelocal.dat");
FSDataOutputStream stm = CreateFile(fs, file1, 1);
stm.Write(fileData);
stm.Close();
try
{
CheckFileContent(uri, file1, fileData, readOffset, shortCircuitUser, conf, shortCircuitFails
);
//RemoteBlockReader have unsupported method read(ByteBuffer bf)
NUnit.Framework.Assert.IsTrue("RemoteBlockReader unsupported method read(ByteBuffer bf) error"
, CheckUnsupportedMethod(fs, file1, fileData, readOffset));
}
catch (IOException e)
{
throw new IOException("doTestShortCircuitReadWithRemoteBlockReader ex error ", e);
}
catch (Exception inEx)
{
throw;
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSkipWithVerifyChecksum()
{
int size = blockSize;
Configuration conf = new Configuration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "/tmp/testSkipWithVerifyChecksum._PORT"
);
DomainSocket.DisableBindPathValidation();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(
true).Build();
FileSystem fs = cluster.GetFileSystem();
try
{
// check that / exists
Path path = new Path("/");
NUnit.Framework.Assert.IsTrue("/ should be a directory", fs.GetFileStatus(path).IsDirectory
() == true);
byte[] fileData = AppendTestUtil.RandomBytes(seed, size * 3);
// create a new file in home directory. Do not close it.
Path file1 = new Path("filelocal.dat");
FSDataOutputStream stm = CreateFile(fs, file1, 1);
// write to file
stm.Write(fileData);
stm.Close();
// now test the skip function
FSDataInputStream instm = fs.Open(file1);
byte[] actual = new byte[fileData.Length];
// read something from the block first, otherwise BlockReaderLocal.skip()
// will not be invoked
int nread = instm.Read(actual, 0, 3);
long skipped = 2 * size + 3;
instm.Seek(skipped);
nread = instm.Read(actual, (int)(skipped + nread), 3);
instm.Close();
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
