/// <summary>Test the updation of NeededReplications for the Appended Block</summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpdateNeededReplicationsForAppendedFile()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
DistributedFileSystem fileSystem = null;
try
{
// create a file.
fileSystem = cluster.GetFileSystem();
Path f = new Path("/testAppend");
FSDataOutputStream create = fileSystem.Create(f, (short)2);
create.Write(Sharpen.Runtime.GetBytesForString("/testAppend"));
create.Close();
// Append to the file.
FSDataOutputStream append = fileSystem.Append(f);
append.Write(Sharpen.Runtime.GetBytesForString("/testAppend"));
append.Close();
// Start a new datanode
cluster.StartDataNodes(conf, 1, true, null, null);
// Check for replications
DFSTestUtil.WaitReplication(fileSystem, f, (short)2);
}
finally
{
if (null != fileSystem)
{
fileSystem.Close();
}
cluster.Shutdown();
}
}
public virtual void TestShortCircuitTraceHooks()
{
Assume.AssumeTrue(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows);
conf = new Configuration();
conf.Set(DFSConfigKeys.DfsClientHtracePrefix + SpanReceiverHost.SpanReceiversConfSuffix
, typeof(TestTracing.SetSpanReceiver).FullName);
conf.SetLong("dfs.blocksize", 100 * 1024);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "testShortCircuitTraceHooks._PORT"
);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
dfs = cluster.GetFileSystem();
try
{
DFSTestUtil.CreateFile(dfs, TestPath, TestLength, (short)1, 5678L);
TraceScope ts = Trace.StartSpan("testShortCircuitTraceHooks", Sampler.Always);
FSDataInputStream stream = dfs.Open(TestPath);
byte[] buf = new byte[TestLength];
IOUtils.ReadFully(stream, buf, 0, TestLength);
stream.Close();
ts.Close();
string[] expectedSpanNames = new string[] { "OpRequestShortCircuitAccessProto", "ShortCircuitShmRequestProto" };
TestTracing.AssertSpanNamesFound(expectedSpanNames);
}
finally
{
dfs.Close();
cluster.Shutdown();
}
}
public virtual void ShutDownCluster()
{
client.Close();
fs.Close();
cluster.ShutdownDataNodes();
cluster.Shutdown();
}
public virtual void TestHedgedReadLoopTooManyTimes()
{
Configuration conf = new Configuration();
int numHedgedReadPoolThreads = 5;
int hedgedReadTimeoutMillis = 50;
conf.SetInt(DFSConfigKeys.DfsDfsclientHedgedReadThreadpoolSize, numHedgedReadPoolThreads
);
conf.SetLong(DFSConfigKeys.DfsDfsclientHedgedReadThresholdMillis, hedgedReadTimeoutMillis
);
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 0);
// Set up the InjectionHandler
DFSClientFaultInjector.instance = Org.Mockito.Mockito.Mock <DFSClientFaultInjector
>();
DFSClientFaultInjector injector = DFSClientFaultInjector.instance;
int sleepMs = 100;
Org.Mockito.Mockito.DoAnswer(new _Answer_296(hedgedReadTimeoutMillis, sleepMs)).When
(injector).FetchFromDatanodeException();
Org.Mockito.Mockito.DoAnswer(new _Answer_309(sleepMs)).When(injector).ReadFromDatanodeDelay
();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format(
true).Build();
DistributedFileSystem fileSys = cluster.GetFileSystem();
DFSClient dfsClient = fileSys.GetClient();
FSDataOutputStream output = null;
DFSInputStream input = null;
string filename = "/hedgedReadMaxOut.dat";
try
{
Path file = new Path(filename);
output = fileSys.Create(file, (short)2);
byte[] data = new byte[64 * 1024];
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Close();
byte[] buffer = new byte[64 * 1024];
input = dfsClient.Open(filename);
input.Read(0, buffer, 0, 1024);
input.Close();
NUnit.Framework.Assert.AreEqual(3, input.GetHedgedReadOpsLoopNumForTesting());
}
catch (BlockMissingException)
{
NUnit.Framework.Assert.IsTrue(false);
}
finally
{
Org.Mockito.Mockito.Reset(injector);
IOUtils.Cleanup(null, input);
IOUtils.Cleanup(null, output);
fileSys.Close();
cluster.Shutdown();
}
}
public virtual void ShutDownCluster()
{
if (cluster != null)
{
fs.Close();
cluster.Shutdown();
cluster = null;
}
}
public virtual void TearDown()
{
fs.Delete(Path, false);
if (cluster != null)
{
fs.Close();
cluster.Shutdown();
cluster = null;
}
}
public static void TearDown()
{
if (hdfs != null)
{
hdfs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
public virtual void ClusterShutdown()
{
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
public virtual void TestRollbackWithQJM()
{
Configuration conf = new HdfsConfiguration();
MiniJournalCluster mjc = null;
MiniDFSCluster cluster = null;
Path foo = new Path("/foo");
Path bar = new Path("/bar");
try
{
mjc = new MiniJournalCluster.Builder(conf).NumJournalNodes(NumJournalNodes).Build
();
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, mjc.GetQuorumJournalURI(JournalId)
.ToString());
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
DFSAdmin dfsadmin = new DFSAdmin(conf);
dfs.Mkdirs(foo);
// start rolling upgrade
dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
NUnit.Framework.Assert.AreEqual(0, dfsadmin.Run(new string[] { "-rollingUpgrade",
"prepare" }));
dfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
// create new directory
dfs.Mkdirs(bar);
dfs.Close();
// rollback
cluster.RestartNameNode("-rollingUpgrade", "rollback");
// make sure /foo is still there, but /bar is not
dfs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(dfs.Exists(foo));
NUnit.Framework.Assert.IsFalse(dfs.Exists(bar));
// check storage in JNs
for (int i = 0; i < NumJournalNodes; i++)
{
FilePath dir = mjc.GetCurrentDir(0, JournalId);
// segments:(startSegment, mkdir, endSegment), (startSegment, upgrade
// marker, mkdir, endSegment)
CheckJNStorage(dir, 4, 7);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
if (mjc != null)
{
mjc.Shutdown();
}
}
}
public static void TearDown()
{
AppendTestUtil.Log.Info("tearDown()");
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
public virtual void ShutDownCluster()
{
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.ShutdownDataNodes();
cluster.Shutdown();
}
}
public virtual void TestFallbackRead()
{
HdfsConfiguration conf = InitZeroCopyTest();
MiniDFSCluster cluster = null;
Path TestPath = new Path("/a");
int TestFileLength = 16385;
int RandomSeed = 23453;
FSDataInputStream fsIn = null;
DistributedFileSystem fs = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
byte[] original = new byte[TestFileLength];
IOUtils.ReadFully(fsIn, original, 0, TestFileLength);
fsIn.Close();
fsIn = fs.Open(TestPath);
TestFallbackImpl(fsIn, original);
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestIsInSafemode()
{
// Check for the standby nn without client failover.
NameNode nn2 = cluster.GetNameNode(1);
NUnit.Framework.Assert.IsTrue("nn2 should be in standby state", nn2.IsStandbyState
());
IPEndPoint nameNodeAddress = nn2.GetNameNodeAddress();
Configuration conf = new Configuration();
DistributedFileSystem dfs = new DistributedFileSystem();
try
{
dfs.Initialize(URI.Create("hdfs://" + nameNodeAddress.GetHostName() + ":" + nameNodeAddress
.Port), conf);
dfs.IsInSafeMode();
NUnit.Framework.Assert.Fail("StandBy should throw exception for isInSafeMode");
}
catch (IOException e)
{
if (e is RemoteException)
{
IOException sbExcpetion = ((RemoteException)e).UnwrapRemoteException();
NUnit.Framework.Assert.IsTrue("StandBy nn should not support isInSafeMode", sbExcpetion
is StandbyException);
}
else
{
throw;
}
}
finally
{
if (null != dfs)
{
dfs.Close();
}
}
// Check with Client FailOver
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
cluster.GetNameNodeRpc(1).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter,
false);
DistributedFileSystem dfsWithFailOver = (DistributedFileSystem)fs;
NUnit.Framework.Assert.IsTrue("ANN should be in SafeMode", dfsWithFailOver.IsInSafeMode
());
cluster.GetNameNodeRpc(1).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave,
false);
NUnit.Framework.Assert.IsFalse("ANN should be out of SafeMode", dfsWithFailOver.IsInSafeMode
());
}
private void CloseDfs(DistributedFileSystem dfs)
{
try
{
if (dfs != null)
{
dfs.Close();
}
}
catch (Exception ignored)
{
// nothing we can do
Sharpen.Runtime.PrintStackTrace(ignored);
}
}
public virtual void TestBootstrapStandbyWithActiveNN()
{
// make nn0 active
cluster.TransitionToActive(0);
// do ops and generate in-progress edit log data
Configuration confNN1 = cluster.GetConfiguration(1);
DistributedFileSystem dfs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs
(cluster, confNN1);
for (int i = 1; i <= 10; i++)
{
dfs.Mkdirs(new Path("/test" + i));
}
dfs.Close();
// shutdown nn1 and delete its edit log files
cluster.ShutdownNameNode(1);
DeleteEditLogIfExists(confNN1);
cluster.GetNameNodeRpc(0).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter,
true);
cluster.GetNameNodeRpc(0).SaveNamespace();
cluster.GetNameNodeRpc(0).SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave,
true);
// check without -skipSharedEditsCheck, Bootstrap should fail for BKJM
// immediately after saveNamespace
int rc = BootstrapStandby.Run(new string[] { "-force", "-nonInteractive" }, confNN1
);
NUnit.Framework.Assert.AreEqual("Mismatches return code", 6, rc);
// check with -skipSharedEditsCheck
rc = BootstrapStandby.Run(new string[] { "-force", "-nonInteractive", "-skipSharedEditsCheck" }, confNN1);
NUnit.Framework.Assert.AreEqual("Mismatches return code", 0, rc);
// Checkpoint as fast as we can, in a tight loop.
confNN1.SetInt(DFSConfigKeys.DfsNamenodeCheckpointPeriodKey, 1);
cluster.RestartNameNode(1);
cluster.TransitionToStandby(1);
NameNode nn0 = cluster.GetNameNode(0);
HATestUtil.WaitForStandbyToCatchUp(nn0, cluster.GetNameNode(1));
long expectedCheckpointTxId = NameNodeAdapter.GetNamesystem(nn0).GetFSImage().GetMostRecentCheckpointTxId
();
HATestUtil.WaitForCheckpoint(cluster, 1, ImmutableList.Of((int)expectedCheckpointTxId
));
// Should have copied over the namespace
FSImageTestUtil.AssertNNHasCheckpoints(cluster, 1, ImmutableList.Of((int)expectedCheckpointTxId
));
FSImageTestUtil.AssertNNFilesMatch(cluster);
}
public virtual void TestAppend2AfterSoftLimit()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1);
//Set small soft-limit for lease
long softLimit = 1L;
long hardLimit = 9999999L;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.SetLeasePeriod(softLimit, hardLimit);
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
DistributedFileSystem fs2 = new DistributedFileSystem();
fs2.Initialize(fs.GetUri(), conf);
Path testPath = new Path("/testAppendAfterSoftLimit");
byte[] fileContents = AppendTestUtil.InitBuffer(32);
// create a new file without closing
FSDataOutputStream @out = fs.Create(testPath);
@out.Write(fileContents);
//Wait for > soft-limit
Sharpen.Thread.Sleep(250);
try
{
FSDataOutputStream appendStream2 = fs2.Append(testPath, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null);
appendStream2.Write(fileContents);
appendStream2.Close();
NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen
());
// make sure we now have 1 block since the first writer was revoked
LocatedBlocks blks = fs.GetClient().GetLocatedBlocks(testPath.ToString(), 0L);
NUnit.Framework.Assert.AreEqual(1, blks.GetLocatedBlocks().Count);
foreach (LocatedBlock blk in blks.GetLocatedBlocks())
{
NUnit.Framework.Assert.AreEqual(fileContents.Length, blk.GetBlockSize());
}
}
finally
{
fs.Close();
fs2.Close();
cluster.Shutdown();
}
}
public virtual void TestMkdir()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
DistributedFileSystem dfs = cluster.GetFileSystem();
try
{
// Create a dir in root dir, should succeed
NUnit.Framework.Assert.IsTrue(dfs.Mkdir(new Path("/mkdir-" + Time.Now()), FsPermission
.GetDefault()));
// Create a dir when parent dir exists as a file, should fail
IOException expectedException = null;
string filePath = "/mkdir-file-" + Time.Now();
DFSTestUtil.WriteFile(dfs, new Path(filePath), "hello world");
try
{
dfs.Mkdir(new Path(filePath + "/mkdir"), FsPermission.GetDefault());
}
catch (IOException e)
{
expectedException = e;
}
NUnit.Framework.Assert.IsTrue("Create a directory when parent dir exists as file using"
+ " mkdir() should throw ParentNotDirectoryException ", expectedException != null &&
expectedException is ParentNotDirectoryException);
// Create a dir in a non-exist directory, should fail
expectedException = null;
try
{
dfs.Mkdir(new Path("/non-exist/mkdir-" + Time.Now()), FsPermission.GetDefault());
}
catch (IOException e)
{
expectedException = e;
}
NUnit.Framework.Assert.IsTrue("Create a directory in a non-exist parent dir using"
+ " mkdir() should throw FileNotFoundException ", expectedException != null &&
expectedException is FileNotFoundException);
}
finally
{
dfs.Close();
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestDowngrade()
{
Configuration conf = new HdfsConfiguration();
MiniQJMHACluster cluster = null;
Path foo = new Path("/foo");
Path bar = new Path("/bar");
try
{
cluster = new MiniQJMHACluster.Builder(conf).Build();
MiniDFSCluster dfsCluster = cluster.GetDfsCluster();
dfsCluster.WaitActive();
// let NN1 tail editlog every 1s
dfsCluster.GetConfiguration(1).SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
dfsCluster.RestartNameNode(1);
dfsCluster.TransitionToActive(0);
DistributedFileSystem dfs = dfsCluster.GetFileSystem(0);
dfs.Mkdirs(foo);
// start rolling upgrade
RollingUpgradeInfo info = dfs.RollingUpgrade(HdfsConstants.RollingUpgradeAction.Prepare
);
NUnit.Framework.Assert.IsTrue(info.IsStarted());
dfs.Mkdirs(bar);
TestRollingUpgrade.QueryForPreparation(dfs);
dfs.Close();
dfsCluster.RestartNameNode(0, true, "-rollingUpgrade", "downgrade");
// Once downgraded, there should be no more fsimage for rollbacks.
NUnit.Framework.Assert.IsFalse(dfsCluster.GetNamesystem(0).GetFSImage().HasRollbackFSImage
());
// shutdown NN1
dfsCluster.ShutdownNameNode(1);
dfsCluster.TransitionToActive(0);
dfs = dfsCluster.GetFileSystem(0);
NUnit.Framework.Assert.IsTrue(dfs.Exists(foo));
NUnit.Framework.Assert.IsTrue(dfs.Exists(bar));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Is the HDFS healthy?
/// HDFS is considered as healthy if it is up and not in safemode.
/// </summary>
/// <param name="uri">the HDFS URI. Note that the URI path is ignored.</param>
/// <returns>true if HDFS is healthy; false, otherwise.</returns>
public static bool IsHealthy(URI uri)
{
//check scheme
string scheme = uri.GetScheme();
if (!Sharpen.Runtime.EqualsIgnoreCase(HdfsConstants.HdfsUriScheme, scheme))
{
throw new ArgumentException("The scheme is not " + HdfsConstants.HdfsUriScheme +
", uri=" + uri);
}
Configuration conf = new Configuration();
//disable FileSystem cache
conf.SetBoolean(string.Format("fs.%s.impl.disable.cache", scheme), true);
//disable client retry for rpc connection and rpc calls
conf.SetBoolean(DFSConfigKeys.DfsClientRetryPolicyEnabledKey, false);
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectMaxRetriesKey, 0);
DistributedFileSystem fs = null;
try
{
fs = (DistributedFileSystem)FileSystem.Get(uri, conf);
bool safemode = fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeGet);
if (Log.IsDebugEnabled())
{
Log.Debug("Is namenode in safemode? " + safemode + "; uri=" + uri);
}
fs.Close();
fs = null;
return(!safemode);
}
catch (IOException e)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Got an exception for uri=" + uri, e);
}
return(false);
}
finally
{
IOUtils.Cleanup(Log, fs);
}
}
/// <summary>
/// Check the blocks of dst file are cleaned after rename with overwrite
/// Restart NN to check the rename successfully
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestRenameWithOverwrite()
{
short replFactor = 2;
long blockSize = 512;
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(replFactor
).Build();
DistributedFileSystem dfs = cluster.GetFileSystem();
try
{
long fileLen = blockSize * 3;
string src = "/foo/src";
string dst = "/foo/dst";
Path srcPath = new Path(src);
Path dstPath = new Path(dst);
DFSTestUtil.CreateFile(dfs, srcPath, fileLen, replFactor, 1);
DFSTestUtil.CreateFile(dfs, dstPath, fileLen, replFactor, 1);
LocatedBlocks lbs = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), dst,
0, fileLen);
BlockManager bm = NameNodeAdapter.GetNamesystem(cluster.GetNameNode()).GetBlockManager
();
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(lbs.GetLocatedBlocks()[0].GetBlock
().GetLocalBlock()) != null);
dfs.Rename(srcPath, dstPath, Options.Rename.Overwrite);
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(lbs.GetLocatedBlocks()[0].GetBlock
().GetLocalBlock()) == null);
// Restart NN and check the rename successfully
cluster.RestartNameNodes();
NUnit.Framework.Assert.IsFalse(dfs.Exists(srcPath));
NUnit.Framework.Assert.IsTrue(dfs.Exists(dstPath));
}
finally
{
if (dfs != null)
{
dfs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void Setup()
{
Configuration conf = new Configuration();
// Turn off IPC client caching, so that the suite can handle
// the restart of the daemons between test cases.
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
MiniQJMHACluster miniQjmHaCluster = new MiniQJMHACluster.Builder(conf).Build();
cluster = miniQjmHaCluster.GetDfsCluster();
jCluster = miniQjmHaCluster.GetJournalCluster();
// make nn0 active
cluster.TransitionToActive(0);
// do sth to generate in-progress edit log data
DistributedFileSystem dfs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs
(cluster, conf);
dfs.Mkdirs(new Path("/test2"));
dfs.Close();
}
public virtual void TestAppendAfterSoftLimit()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1);
conf.SetBoolean(DFSConfigKeys.DfsSupportAppendKey, true);
//Set small soft-limit for lease
long softLimit = 1L;
long hardLimit = 9999999L;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.SetLeasePeriod(softLimit, hardLimit);
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
FileSystem fs2 = new DistributedFileSystem();
fs2.Initialize(fs.GetUri(), conf);
Path testPath = new Path("/testAppendAfterSoftLimit");
byte[] fileContents = AppendTestUtil.InitBuffer(32);
// create a new file without closing
FSDataOutputStream @out = fs.Create(testPath);
@out.Write(fileContents);
//Wait for > soft-limit
Sharpen.Thread.Sleep(250);
try
{
FSDataOutputStream appendStream2 = fs2.Append(testPath);
appendStream2.Write(fileContents);
appendStream2.Close();
NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen
());
}
finally
{
fs.Close();
fs2.Close();
cluster.Shutdown();
}
}
public virtual void TestAppend2Twice()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem fs1 = cluster.GetFileSystem();
FileSystem fs2 = AppendTestUtil.CreateHdfsWithDifferentUsername(conf);
try
{
Path p = new Path("/testAppendTwice/foo");
int len = 1 << 16;
byte[] fileContents = AppendTestUtil.InitBuffer(len);
{
// create a new file with a full block.
FSDataOutputStream @out = fs2.Create(p, true, 4096, (short)1, len);
@out.Write(fileContents, 0, len);
@out.Close();
}
//1st append does not add any data so that the last block remains full
//and the last block in INodeFileUnderConstruction is a BlockInfo
//but not BlockInfoUnderConstruction.
((DistributedFileSystem)fs2).Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock
), 4096, null);
// 2nd append should get AlreadyBeingCreatedException
fs1.Append(p);
NUnit.Framework.Assert.Fail();
}
catch (RemoteException re)
{
AppendTestUtil.Log.Info("Got an exception:", re);
NUnit.Framework.Assert.AreEqual(typeof(AlreadyBeingCreatedException).FullName, re
.GetClassName());
}
finally
{
fs2.Close();
fs1.Close();
cluster.Shutdown();
}
}
public virtual void TestBlockMissingException()
{
Log.Info("Test testBlockMissingException started.");
long blockSize = 1024L;
int numBlocks = 4;
conf = new HdfsConfiguration();
// Set short retry timeouts so this test runs faster
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10);
try
{
dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDatanodes).Build();
dfs.WaitActive();
fileSys = dfs.GetFileSystem();
Path file1 = new Path("/user/dhruba/raidtest/file1");
CreateOldFile(fileSys, file1, 1, numBlocks, blockSize);
// extract block locations from File system. Wait till file is closed.
LocatedBlocks locations = null;
locations = fileSys.dfs.GetNamenode().GetBlockLocations(file1.ToString(), 0, numBlocks
* blockSize);
// remove block of file
Log.Info("Remove first block of file");
CorruptBlock(file1, locations.Get(0).GetBlock());
// validate that the system throws BlockMissingException
ValidateFile(fileSys, file1);
}
finally
{
if (fileSys != null)
{
fileSys.Close();
}
if (dfs != null)
{
dfs.Shutdown();
}
}
Log.Info("Test testBlockMissingException completed.");
}
public virtual void ShutDownCluster()
{
// Dump all RamDisk JMX metrics before shutdown the cluster
PrintRamDiskJMXMetrics();
if (fs != null)
{
fs.Close();
fs = null;
client = null;
}
if (cluster != null)
{
cluster.ShutdownDataNodes();
cluster.Shutdown();
cluster = null;
}
if (jmx != null)
{
jmx = null;
}
IOUtils.CloseQuietly(sockDir);
sockDir = null;
}
public virtual void HSyncUpdateLength_00()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
DistributedFileSystem fileSystem = cluster.GetFileSystem();
try
{
Path path = new Path(fName);
FSDataOutputStream stm = fileSystem.Create(path, true, 4096, (short)2, AppendTestUtil
.BlockSize);
System.Console.Out.WriteLine("Created file " + path.ToString());
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
long currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(0L, currentFileLength);
stm.Close();
}
finally
{
fileSystem.Close();
cluster.Shutdown();
}
}
/// <summary>
/// The method starts new cluster with defined Configuration; creates a file
/// with specified block_size and writes 10 equal sections in it; it also calls
/// hflush/hsync after each write and throws an IOException in case of an error.
/// </summary>
/// <param name="conf">cluster configuration</param>
/// <param name="fileName">of the file to be created and processed as required</param>
/// <param name="block_size">value to be used for the file's creation</param>
/// <param name="replicas">is the number of replicas</param>
/// <param name="isSync">hsync or hflush</param>
/// <param name="syncFlags">specify the semantic of the sync/flush</param>
/// <exception cref="System.IO.IOException">in case of any errors</exception>
public static void DoTheJob(Configuration conf, string fileName, long block_size,
short replicas, bool isSync, EnumSet <HdfsDataOutputStream.SyncFlag> syncFlags)
{
byte[] fileContent;
int Sections = 10;
fileContent = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(replicas).
Build();
// Make sure we work with DFS in order to utilize all its functionality
DistributedFileSystem fileSystem = cluster.GetFileSystem();
FSDataInputStream @is;
try
{
Path path = new Path(fileName);
string pathName = new Path(fileSystem.GetWorkingDirectory(), path).ToUri().GetPath
();
FSDataOutputStream stm = fileSystem.Create(path, false, 4096, replicas, block_size
);
System.Console.Out.WriteLine("Created file " + fileName);
int tenth = AppendTestUtil.FileSize / Sections;
int rounding = AppendTestUtil.FileSize - tenth * Sections;
for (int i = 0; i < Sections; i++)
{
System.Console.Out.WriteLine("Writing " + (tenth * i) + " to " + (tenth * (i + 1)
) + " section to file " + fileName);
// write to the file
stm.Write(fileContent, tenth * i, tenth);
// Wait while hflush/hsync pushes all packets through built pipeline
if (isSync)
{
((DFSOutputStream)stm.GetWrappedStream()).Hsync(syncFlags);
}
else
{
((DFSOutputStream)stm.GetWrappedStream()).Hflush();
}
// Check file length if updatelength is required
if (isSync && syncFlags.Contains(HdfsDataOutputStream.SyncFlag.UpdateLength))
{
long currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual("File size doesn't match for hsync/hflush with updating the length"
, tenth * (i + 1), currentFileLength);
}
else
{
if (isSync && syncFlags.Contains(HdfsDataOutputStream.SyncFlag.EndBlock))
{
LocatedBlocks blocks = fileSystem.dfs.GetLocatedBlocks(pathName, 0);
NUnit.Framework.Assert.AreEqual(i + 1, blocks.GetLocatedBlocks().Count);
}
}
byte[] toRead = new byte[tenth];
byte[] expected = new byte[tenth];
System.Array.Copy(fileContent, tenth * i, expected, 0, tenth);
// Open the same file for read. Need to create new reader after every write operation(!)
@is = fileSystem.Open(path);
@is.Seek(tenth * i);
int readBytes = @is.Read(toRead, 0, tenth);
System.Console.Out.WriteLine("Has read " + readBytes);
NUnit.Framework.Assert.IsTrue("Should've get more bytes", (readBytes > 0) && (readBytes
<= tenth));
@is.Close();
CheckData(toRead, 0, readBytes, expected, "Partial verification");
}
System.Console.Out.WriteLine("Writing " + (tenth * Sections) + " to " + (tenth *
Sections + rounding) + " section to file " + fileName);
stm.Write(fileContent, tenth * Sections, rounding);
stm.Close();
NUnit.Framework.Assert.AreEqual("File size doesn't match ", AppendTestUtil.FileSize
, fileSystem.GetFileStatus(path).GetLen());
AppendTestUtil.CheckFullFile(fileSystem, path, fileContent.Length, fileContent, "hflush()"
);
}
finally
{
fileSystem.Close();
cluster.Shutdown();
}
}
/// <summary>
/// When an InterruptedException is sent to a thread calling
/// FileChannel#read, the FileChannel is immediately closed and the
/// thread gets an exception.
/// </summary>
/// <remarks>
/// When an InterruptedException is sent to a thread calling
/// FileChannel#read, the FileChannel is immediately closed and the
/// thread gets an exception. This effectively means that we might have
/// someone asynchronously calling close() on the file descriptors we use
/// in BlockReaderLocal. So when unreferencing a ShortCircuitReplica in
/// ShortCircuitCache#unref, we should check if the FileChannel objects
/// are still open. If not, we should purge the replica to avoid giving
/// it out to any future readers.
/// This is a regression test for HDFS-6227: Short circuit read failed
/// due to ClosedChannelException.
/// Note that you may still get ClosedChannelException errors if two threads
/// are reading from the same replica and an InterruptedException is delivered
/// to one of them.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestPurgingClosedReplicas()
{
BlockReaderTestUtil.EnableBlockReaderFactoryTracing();
AtomicInteger replicasCreated = new AtomicInteger(0);
AtomicBoolean testFailed = new AtomicBoolean(false);
DFSInputStream.tcpReadsDisabledForTesting = true;
BlockReaderFactory.createShortCircuitReplicaInfoCallback = new _ShortCircuitReplicaCreator_443
(replicasCreated);
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
Configuration conf = CreateShortCircuitConf("testPurgingClosedReplicas", sockDir);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string TestFile = "/test_file";
int TestFileLen = 4095;
int Seed = unchecked ((int)(0xFADE0));
DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI(0
), conf);
DFSTestUtil.CreateFile(fs, new Path(TestFile), TestFileLen, (short)1, Seed);
Semaphore sem = Sharpen.Extensions.CreateSemaphore(0);
IList <LocatedBlock> locatedBlocks = cluster.GetNameNode().GetRpcServer().GetBlockLocations
(TestFile, 0, TestFileLen).GetLocatedBlocks();
LocatedBlock lblock = locatedBlocks[0];
// first block
byte[] buf = new byte[TestFileLen];
Runnable readerRunnable = new _Runnable_471(cluster, lblock, TestFileLen, sem, buf
, testFailed);
Sharpen.Thread thread = new Sharpen.Thread(readerRunnable);
thread.Start();
// While the thread is reading, send it interrupts.
// These should trigger a ClosedChannelException.
while (thread.IsAlive())
{
sem.AcquireUninterruptibly();
thread.Interrupt();
sem.Release();
}
NUnit.Framework.Assert.IsFalse(testFailed.Get());
// We should be able to read from the file without
// getting a ClosedChannelException.
BlockReader blockReader = null;
try
{
blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen);
blockReader.ReadFully(buf, 0, TestFileLen);
}
finally
{
if (blockReader != null)
{
blockReader.Close();
}
}
byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
NUnit.Framework.Assert.IsTrue(Arrays.Equals(buf, expected));
// Another ShortCircuitReplica object should have been created.
NUnit.Framework.Assert.AreEqual(2, replicasCreated.Get());
dfs.Close();
cluster.Shutdown();
sockDir.Close();
}
public virtual void ShutDownCluster()
{
dfs.Close();
cluster.Shutdown();
}
public virtual void TestSimpleAppend2()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsDatanodeHandlerCountKey, 50);
fileContents = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem fs = cluster.GetFileSystem();
try
{
{
// test appending to a file.
// create a new file.
Path file1 = new Path("/simpleAppend.dat");
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
System.Console.Out.WriteLine("Created file simpleAppend.dat");
// write to file
int mid = 186;
// io.bytes.per.checksum bytes
System.Console.Out.WriteLine("Writing " + mid + " bytes to file " + file1);
stm.Write(fileContents, 0, mid);
stm.Close();
System.Console.Out.WriteLine("Wrote and Closed first part of file.");
// write to file
int mid2 = 607;
// io.bytes.per.checksum bytes
System.Console.Out.WriteLine("Writing " + mid + " bytes to file " + file1);
stm = fs.Append(file1, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096,
null);
stm.Write(fileContents, mid, mid2 - mid);
stm.Close();
System.Console.Out.WriteLine("Wrote and Closed second part of file.");
// write the remainder of the file
stm = fs.Append(file1, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096,
null);
// ensure getPos is set to reflect existing size of the file
NUnit.Framework.Assert.IsTrue(stm.GetPos() > 0);
System.Console.Out.WriteLine("Writing " + (AppendTestUtil.FileSize - mid2) + " bytes to file "
+ file1);
stm.Write(fileContents, mid2, AppendTestUtil.FileSize - mid2);
System.Console.Out.WriteLine("Written second part of file");
stm.Close();
System.Console.Out.WriteLine("Wrote and Closed second part of file.");
// verify that entire file is good
AppendTestUtil.CheckFullFile(fs, file1, AppendTestUtil.FileSize, fileContents, "Read 2"
);
// also make sure there three different blocks for the file
IList <LocatedBlock> blocks = fs.GetClient().GetLocatedBlocks(file1.ToString(), 0L
).GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(12, blocks.Count);
// the block size is 1024
NUnit.Framework.Assert.AreEqual(mid, blocks[0].GetBlockSize());
NUnit.Framework.Assert.AreEqual(mid2 - mid, blocks[1].GetBlockSize());
for (int i = 2; i < 11; i++)
{
NUnit.Framework.Assert.AreEqual(AppendTestUtil.BlockSize, blocks[i].GetBlockSize(
));
}
NUnit.Framework.Assert.AreEqual((AppendTestUtil.FileSize - mid2) % AppendTestUtil
.BlockSize, blocks[11].GetBlockSize());
}
{
// test appending to an non-existing file.
FSDataOutputStream @out = null;
try
{
@out = fs.Append(new Path("/non-existing.dat"), EnumSet.Of(CreateFlag.Append, CreateFlag
.NewBlock), 4096, null);
NUnit.Framework.Assert.Fail("Expected to have FileNotFoundException");
}
catch (FileNotFoundException fnfe)
{
System.Console.Out.WriteLine("Good: got " + fnfe);
Sharpen.Runtime.PrintStackTrace(fnfe, System.Console.Out);
}
finally
{
IOUtils.CloseStream(@out);
}
}
{
// test append permission.
// set root to all writable
Path root = new Path("/");
fs.SetPermission(root, new FsPermission((short)0x1ff));
fs.Close();
// login as a different user
UserGroupInformation superuser = UserGroupInformation.GetCurrentUser();
string username = "******";
string group = "testappendgroup";
NUnit.Framework.Assert.IsFalse(superuser.GetShortUserName().Equals(username));
NUnit.Framework.Assert.IsFalse(Arrays.AsList(superuser.GetGroupNames()).Contains(
group));
UserGroupInformation appenduser = UserGroupInformation.CreateUserForTesting(username
, new string[] { group });
fs = (DistributedFileSystem)DFSTestUtil.GetFileSystemAs(appenduser, conf);
// create a file
Path dir = new Path(root, GetType().Name);
Path foo = new Path(dir, "foo.dat");
FSDataOutputStream @out = null;
int offset = 0;
try
{
@out = fs.Create(foo);
int len = 10 + AppendTestUtil.NextInt(100);
@out.Write(fileContents, offset, len);
offset += len;
}
finally
{
IOUtils.CloseStream(@out);
}
// change dir and foo to minimal permissions.
fs.SetPermission(dir, new FsPermission((short)0x40));
fs.SetPermission(foo, new FsPermission((short)0x80));
// try append, should success
@out = null;
try
{
@out = fs.Append(foo, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096, null
);
int len = 10 + AppendTestUtil.NextInt(100);
@out.Write(fileContents, offset, len);
offset += len;
}
finally
{
IOUtils.CloseStream(@out);
}
// change dir and foo to all but no write on foo.
fs.SetPermission(foo, new FsPermission((short)0x17f));
fs.SetPermission(dir, new FsPermission((short)0x1ff));
// try append, should fail
@out = null;
try
{
@out = fs.Append(foo, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096, null
);
NUnit.Framework.Assert.Fail("Expected to have AccessControlException");
}
catch (AccessControlException ace)
{
System.Console.Out.WriteLine("Good: got " + ace);
Sharpen.Runtime.PrintStackTrace(ace, System.Console.Out);
}
finally
{
IOUtils.CloseStream(@out);
}
}
}
finally
{
fs.Close();
cluster.Shutdown();
}
}