public virtual void TestAbandonBlock()
{
string src = FileNamePrefix + "foo";
// Start writing a file but do not close it
FSDataOutputStream fout = fs.Create(new Path(src), true, 4096, (short)1, 512L);
for (int i = 0; i < 1024; i++)
{
fout.Write(123);
}
fout.Hflush();
long fileId = ((DFSOutputStream)fout.GetWrappedStream()).GetFileId();
// Now abandon the last block
DFSClient dfsclient = DFSClientAdapter.GetDFSClient(fs);
LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue
);
int orginalNumBlocks = blocks.LocatedBlockCount();
LocatedBlock b = blocks.GetLastLocatedBlock();
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
);
// call abandonBlock again to make sure the operation is idempotent
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
);
// And close the file
fout.Close();
// Close cluster and check the block has been abandoned after restart
cluster.RestartNameNode();
blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue);
NUnit.Framework.Assert.AreEqual("Blocks " + b + " has not been abandoned.", orginalNumBlocks
, blocks.LocatedBlockCount() + 1);
}
/// <exception cref="System.IO.IOException"/>
private void TestHSyncOperation(bool testWithAppend)
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
DistributedFileSystem fs = cluster.GetFileSystem();
Path p = new Path("/testHSync/foo");
int len = 1 << 16;
FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null);
if (testWithAppend)
{
// re-open the file with append call
@out.Close();
@out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.SyncBlock), 4096, null
);
}
@out.Hflush();
// hflush does not sync
CheckSyncMetric(cluster, 0);
@out.Hsync();
// hsync on empty file does nothing
CheckSyncMetric(cluster, 0);
@out.Write(1);
CheckSyncMetric(cluster, 0);
@out.Hsync();
CheckSyncMetric(cluster, 1);
// avoiding repeated hsyncs is a potential future optimization
@out.Hsync();
CheckSyncMetric(cluster, 2);
@out.Hflush();
// hflush still does not sync
CheckSyncMetric(cluster, 2);
@out.Close();
// close is sync'ing
CheckSyncMetric(cluster, 3);
// same with a file created with out SYNC_BLOCK
@out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag.Create, CreateFlag
.Overwrite), 4096, (short)1, len, null);
@out.Hsync();
CheckSyncMetric(cluster, 3);
@out.Write(1);
CheckSyncMetric(cluster, 3);
@out.Hsync();
CheckSyncMetric(cluster, 4);
// repeated hsyncs
@out.Hsync();
CheckSyncMetric(cluster, 5);
@out.Close();
// close does not sync (not opened with SYNC_BLOCK)
CheckSyncMetric(cluster, 5);
cluster.Shutdown();
}
/// <exception cref="System.IO.IOException"/>
private void TestPersistHelper(Configuration conf)
{
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).Build();
cluster.WaitActive();
FSNamesystem fsn = cluster.GetNamesystem();
DistributedFileSystem fs = cluster.GetFileSystem();
Path dir = new Path("/abc/def");
Path file1 = new Path(dir, "f1");
Path file2 = new Path(dir, "f2");
// create an empty file f1
fs.Create(file1).Close();
// create an under-construction file f2
FSDataOutputStream @out = fs.Create(file2);
@out.WriteBytes("hello");
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
// checkpoint
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
fs.SaveNamespace();
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
cluster.RestartNameNode();
cluster.WaitActive();
fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(fs.IsDirectory(dir));
NUnit.Framework.Assert.IsTrue(fs.Exists(file1));
NUnit.Framework.Assert.IsTrue(fs.Exists(file2));
// check internals of file2
INodeFile file2Node = fsn.dir.GetINode4Write(file2.ToString()).AsFile();
NUnit.Framework.Assert.AreEqual("hello".Length, file2Node.ComputeFileSize());
NUnit.Framework.Assert.IsTrue(file2Node.IsUnderConstruction());
BlockInfoContiguous[] blks = file2Node.GetBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Length);
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.BlockUCState.UnderConstruction
, blks[0].GetBlockUCState());
// check lease manager
LeaseManager.Lease lease = fsn.leaseManager.GetLeaseByPath(file2.ToString());
NUnit.Framework.Assert.IsNotNull(lease);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Test the updation of NeededReplications for the Appended Block</summary>
/// <exception cref="System.Exception"/>
public virtual void TestUpdateNeededReplicationsForAppendedFile()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
DistributedFileSystem fileSystem = null;
try
{
// create a file.
fileSystem = cluster.GetFileSystem();
Path f = new Path("/testAppend");
FSDataOutputStream create = fileSystem.Create(f, (short)2);
create.Write(Sharpen.Runtime.GetBytesForString("/testAppend"));
create.Close();
// Append to the file.
FSDataOutputStream append = fileSystem.Append(f);
append.Write(Sharpen.Runtime.GetBytesForString("/testAppend"));
append.Close();
// Start a new datanode
cluster.StartDataNodes(conf, 1, true, null, null);
// Check for replications
DFSTestUtil.WaitReplication(fileSystem, f, (short)2);
}
finally
{
if (null != fileSystem)
{
fileSystem.Close();
}
cluster.Shutdown();
}
}
/// <summary>
/// The idea for making sure that there is no more than one instance
/// running in an HDFS is to create a file in the HDFS, writes the hostname
/// of the machine on which the instance is running to the file, but did not
/// close the file until it exits.
/// </summary>
/// <remarks>
/// The idea for making sure that there is no more than one instance
/// running in an HDFS is to create a file in the HDFS, writes the hostname
/// of the machine on which the instance is running to the file, but did not
/// close the file until it exits.
/// This prevents the second instance from running because it can not
/// creates the file while the first one is running.
/// This method checks if there is any running instance. If no, mark yes.
/// Note that this is an atomic operation.
/// </remarks>
/// <returns>
/// null if there is a running instance;
/// otherwise, the output stream to the newly created file.
/// </returns>
/// <exception cref="System.IO.IOException"/>
private OutputStream CheckAndMarkRunning()
{
try
{
if (fs.Exists(idPath))
{
// try appending to it so that it will fail fast if another balancer is
// running.
IOUtils.CloseStream(fs.Append(idPath));
fs.Delete(idPath, true);
}
FSDataOutputStream fsout = fs.Create(idPath, false);
// mark balancer idPath to be deleted during filesystem closure
fs.DeleteOnExit(idPath);
if (write2IdFile)
{
fsout.WriteBytes(Sharpen.Runtime.GetLocalHost().GetHostName());
fsout.Hflush();
}
return(fsout);
}
catch (RemoteException e)
{
if (typeof(AlreadyBeingCreatedException).FullName.Equals(e.GetClassName()))
{
return(null);
}
else
{
throw;
}
}
}
public virtual void TestSaveNamespaceWithRenamedLease()
{
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration()).NumDataNodes
(1).Build();
cluster.WaitActive();
DistributedFileSystem fs = (DistributedFileSystem)cluster.GetFileSystem();
OutputStream @out = null;
try
{
fs.Mkdirs(new Path("/test-target"));
@out = fs.Create(new Path("/test-source/foo"));
// don't close
fs.Rename(new Path("/test-source/"), new Path("/test-target/"));
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
cluster.GetNameNodeRpc().SaveNamespace();
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
}
finally
{
IOUtils.Cleanup(Log, @out, fs);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
protected internal void MakeTestFile(Path path, long length, bool isLazyPersist)
{
EnumSet <CreateFlag> createFlags = EnumSet.Of(CreateFlag.Create);
if (isLazyPersist)
{
createFlags.AddItem(CreateFlag.LazyPersist);
}
FSDataOutputStream fos = null;
try
{
fos = fs.Create(path, FsPermission.GetFileDefault(), createFlags, BufferLength, ReplFactor
, BlockSize, null);
// Allocate a block.
byte[] buffer = new byte[BufferLength];
for (int bytesWritten = 0; bytesWritten < length;)
{
fos.Write(buffer, 0, buffer.Length);
bytesWritten += buffer.Length;
}
if (length > 0)
{
fos.Hsync();
}
}
finally
{
IOUtils.CloseQuietly(fos);
}
}
/// <summary>
/// Test fsimage loading when 1) there is an empty file loaded from fsimage,
/// and 2) there is later an append operation to be applied from edit log.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestLoadImageWithEmptyFile()
{
// create an empty file
Path file = new Path(dir, "file");
FSDataOutputStream @out = hdfs.Create(file);
@out.Close();
// save namespace
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
hdfs.SaveNamespace();
hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
// append to the empty file
@out = hdfs.Append(file);
@out.Write(1);
@out.Close();
// restart cluster
cluster.Shutdown();
cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication
).Build();
cluster.WaitActive();
hdfs = cluster.GetFileSystem();
FileStatus status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(1, status.GetLen());
}
/// <exception cref="System.IO.IOException"/>
internal SlowWriter(DistributedFileSystem fs, Path filepath, long sleepms)
: base(typeof(TestReplaceDatanodeOnFailure.SlowWriter).Name + ":" + filepath)
{
this.filepath = filepath;
this.@out = (HdfsDataOutputStream)fs.Create(filepath, Replication);
this.sleepms = sleepms;
}
public virtual void TestMoverFailedRetry()
{
// HDFS-8147
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsMoverRetryMaxAttemptsKey, "2");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).StorageTypes
(new StorageType[][] { new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType
[] { StorageType.Disk, StorageType.Archive } }).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string file = "/testMoverFailedRetry";
// write to DISK
FSDataOutputStream @out = dfs.Create(new Path(file), (short)2);
@out.WriteChars("testMoverFailedRetry");
@out.Close();
// Delete block file so, block move will fail with FileNotFoundException
LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
cluster.CorruptBlockOnDataNodesByDeletingBlockFile(lb.GetBlock());
// move to ARCHIVE
dfs.SetStoragePolicy(new Path(file), "COLD");
int rc = ToolRunner.Run(conf, new Mover.Cli(), new string[] { "-p", file.ToString
() });
NUnit.Framework.Assert.AreEqual("Movement should fail after some retry", ExitStatus
.IoException.GetExitCode(), rc);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestCloseTwice()
{
DistributedFileSystem fs = cluster.GetFileSystem();
FSDataOutputStream os = fs.Create(new Path("/test"));
DFSOutputStream dos = (DFSOutputStream)Whitebox.GetInternalState(os, "wrappedStream"
);
AtomicReference <IOException> ex = (AtomicReference <IOException>)Whitebox.GetInternalState
(dos, "lastException");
NUnit.Framework.Assert.AreEqual(null, ex.Get());
dos.Close();
IOException dummy = new IOException("dummy");
ex.Set(dummy);
try
{
dos.Close();
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual(e, dummy);
}
NUnit.Framework.Assert.AreEqual(null, ex.Get());
dos.Close();
}
public virtual void TestAppendWithPipelineRecovery()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
FSDataOutputStream @out = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(true).ManageNameDfsDirs
(true).NumDataNodes(4).Racks(new string[] { "/rack1", "/rack1", "/rack2", "/rack2" }).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
Path path = new Path("/test1");
@out = fs.Create(path, true, BlockSize, (short)3, BlockSize);
AppendTestUtil.Write(@out, 0, 1024);
@out.Close();
cluster.StopDataNode(3);
@out = fs.Append(path);
AppendTestUtil.Write(@out, 1024, 1024);
@out.Close();
cluster.RestartNameNode(true);
AppendTestUtil.Check(fs, path, 2048);
}
finally
{
IOUtils.CloseStream(@out);
if (null != cluster)
{
cluster.Shutdown();
}
}
}
public virtual void TestAppendLessThanChecksumChunk()
{
byte[] buf = new byte[1024];
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NumDataNodes
(1).Build();
cluster.WaitActive();
try
{
using (DistributedFileSystem fs = cluster.GetFileSystem())
{
int len1 = 200;
int len2 = 300;
Path p = new Path("/foo");
FSDataOutputStream @out = fs.Create(p);
@out.Write(buf, 0, len1);
@out.Close();
@out = fs.Append(p);
@out.Write(buf, 0, len2);
// flush but leave open
@out.Hflush();
// read data to verify the replica's content and checksum are correct
FSDataInputStream @in = fs.Open(p);
int length = @in.Read(0, buf, 0, len1 + len2);
NUnit.Framework.Assert.IsTrue(length > 0);
@in.Close();
@out.Close();
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestHedgedReadLoopTooManyTimes()
{
Configuration conf = new Configuration();
int numHedgedReadPoolThreads = 5;
int hedgedReadTimeoutMillis = 50;
conf.SetInt(DFSConfigKeys.DfsDfsclientHedgedReadThreadpoolSize, numHedgedReadPoolThreads
);
conf.SetLong(DFSConfigKeys.DfsDfsclientHedgedReadThresholdMillis, hedgedReadTimeoutMillis
);
conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 0);
// Set up the InjectionHandler
DFSClientFaultInjector.instance = Org.Mockito.Mockito.Mock <DFSClientFaultInjector
>();
DFSClientFaultInjector injector = DFSClientFaultInjector.instance;
int sleepMs = 100;
Org.Mockito.Mockito.DoAnswer(new _Answer_296(hedgedReadTimeoutMillis, sleepMs)).When
(injector).FetchFromDatanodeException();
Org.Mockito.Mockito.DoAnswer(new _Answer_309(sleepMs)).When(injector).ReadFromDatanodeDelay
();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format(
true).Build();
DistributedFileSystem fileSys = cluster.GetFileSystem();
DFSClient dfsClient = fileSys.GetClient();
FSDataOutputStream output = null;
DFSInputStream input = null;
string filename = "/hedgedReadMaxOut.dat";
try
{
Path file = new Path(filename);
output = fileSys.Create(file, (short)2);
byte[] data = new byte[64 * 1024];
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Write(data);
output.Flush();
output.Close();
byte[] buffer = new byte[64 * 1024];
input = dfsClient.Open(filename);
input.Read(0, buffer, 0, 1024);
input.Close();
NUnit.Framework.Assert.AreEqual(3, input.GetHedgedReadOpsLoopNumForTesting());
}
catch (BlockMissingException)
{
NUnit.Framework.Assert.IsTrue(false);
}
finally
{
Org.Mockito.Mockito.Reset(injector);
IOUtils.Cleanup(null, input);
IOUtils.Cleanup(null, output);
fileSys.Close();
cluster.Shutdown();
}
}
public virtual void TestOpenFilesWithRename()
{
Path path = new Path("/test");
DoWriteAndAbort(fs, path);
// check for zero sized blocks
Path fileWithEmptyBlock = new Path("/test/test/test4");
fs.Create(fileWithEmptyBlock);
NamenodeProtocols nameNodeRpc = cluster.GetNameNodeRpc();
string clientName = fs.GetClient().GetClientName();
// create one empty block
nameNodeRpc.AddBlock(fileWithEmptyBlock.ToString(), clientName, null, null, INodeId
.GrandfatherInodeId, null);
fs.CreateSnapshot(path, "s2");
fs.Rename(new Path("/test/test"), new Path("/test/test-renamed"));
fs.Delete(new Path("/test/test-renamed"), true);
NameNode nameNode = cluster.GetNameNode();
NameNodeAdapter.EnterSafeMode(nameNode, false);
NameNodeAdapter.SaveNamespace(nameNode);
NameNodeAdapter.LeaveSafeMode(nameNode);
cluster.RestartNameNode(true);
}
public virtual void TestAppend()
{
int maxOldFileLen = 2 * BlockSize + 1;
int maxFlushedBytes = BlockSize;
byte[] contents = AppendTestUtil.InitBuffer(maxOldFileLen + 2 * maxFlushedBytes);
for (int oldFileLen = 0; oldFileLen <= maxOldFileLen; oldFileLen++)
{
for (int flushedBytes1 = 0; flushedBytes1 <= maxFlushedBytes; flushedBytes1++)
{
for (int flushedBytes2 = 0; flushedBytes2 <= maxFlushedBytes; flushedBytes2++)
{
int fileLen = oldFileLen + flushedBytes1 + flushedBytes2;
// create the initial file of oldFileLen
Path p = new Path("foo" + oldFileLen + "_" + flushedBytes1 + "_" + flushedBytes2);
Log.Info("Creating file " + p);
FSDataOutputStream @out = fs.Create(p, false, conf.GetInt(CommonConfigurationKeys
.IoFileBufferSizeKey, 4096), Replication, BlockSize);
@out.Write(contents, 0, oldFileLen);
@out.Close();
// append flushedBytes bytes to the file
@out = fs.Append(p);
@out.Write(contents, oldFileLen, flushedBytes1);
@out.Hflush();
// write another flushedBytes2 bytes to the file
@out.Write(contents, oldFileLen + flushedBytes1, flushedBytes2);
@out.Close();
// validate the file content
AppendTestUtil.CheckFullFile(fs, p, fileLen, contents, p.ToString());
fs.Delete(p, false);
}
}
}
}
/// <exception cref="System.IO.IOException"/>
private void TestXAttr(bool persistNamespace)
{
Path path = new Path("/p");
DistributedFileSystem fs = cluster.GetFileSystem();
fs.Create(path).Close();
fs.SetXAttr(path, name1, value1, EnumSet.Of(XAttrSetFlag.Create));
fs.SetXAttr(path, name2, value2, EnumSet.Of(XAttrSetFlag.Create));
fs.SetXAttr(path, name3, null, EnumSet.Of(XAttrSetFlag.Create));
Restart(fs, persistNamespace);
IDictionary <string, byte[]> xattrs = fs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 3);
Assert.AssertArrayEquals(value1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
Assert.AssertArrayEquals(value3, xattrs[name3]);
fs.SetXAttr(path, name1, newValue1, EnumSet.Of(XAttrSetFlag.Replace));
Restart(fs, persistNamespace);
xattrs = fs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 3);
Assert.AssertArrayEquals(newValue1, xattrs[name1]);
Assert.AssertArrayEquals(value2, xattrs[name2]);
Assert.AssertArrayEquals(value3, xattrs[name3]);
fs.RemoveXAttr(path, name1);
fs.RemoveXAttr(path, name2);
fs.RemoveXAttr(path, name3);
Restart(fs, persistNamespace);
xattrs = fs.GetXAttrs(path);
NUnit.Framework.Assert.AreEqual(xattrs.Count, 0);
}
public virtual void TestScheduleSameBlock()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string file = "/testScheduleSameBlock/file";
{
FSDataOutputStream @out = dfs.Create(new Path(file));
@out.WriteChars("testScheduleSameBlock");
@out.Close();
}
Org.Apache.Hadoop.Hdfs.Server.Mover.Mover mover = NewMover(conf);
mover.Init();
Mover.Processor processor = new Mover.Processor(this);
LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
IList <Mover.MLocation> locations = Mover.MLocation.ToLocations(lb);
Mover.MLocation ml = locations[0];
Dispatcher.DBlock db = mover.NewDBlock(lb.GetBlock().GetLocalBlock(), locations);
IList <StorageType> storageTypes = new AList <StorageType>(Arrays.AsList(StorageType
.Default, StorageType.Default));
NUnit.Framework.Assert.IsTrue(processor.ScheduleMoveReplica(db, ml, storageTypes)
);
NUnit.Framework.Assert.IsFalse(processor.ScheduleMoveReplica(db, ml, storageTypes
));
}
finally
{
cluster.Shutdown();
}
}
public virtual void HSyncEndBlock_00()
{
int preferredBlockSize = 1024;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, preferredBlockSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
DistributedFileSystem fileSystem = cluster.GetFileSystem();
FSDataOutputStream stm = null;
try
{
Path path = new Path("/" + fName);
stm = fileSystem.Create(path, true, 4096, (short)2, AppendTestUtil.BlockSize);
System.Console.Out.WriteLine("Created file " + path.ToString());
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.EndBlock));
long currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(0L, currentFileLength);
LocatedBlocks blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(0, blocks.GetLocatedBlocks().Count);
// write a block and call hsync(end_block) at the block boundary
stm.Write(new byte[preferredBlockSize]);
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.EndBlock));
currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(preferredBlockSize, currentFileLength);
blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(1, blocks.GetLocatedBlocks().Count);
// call hsync then call hsync(end_block) immediately
stm.Write(new byte[preferredBlockSize / 2]);
stm.Hsync();
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.EndBlock));
currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2, currentFileLength
);
blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(2, blocks.GetLocatedBlocks().Count);
stm.Write(new byte[preferredBlockSize / 4]);
stm.Hsync();
currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2 + preferredBlockSize
/ 4, currentFileLength);
blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(3, blocks.GetLocatedBlocks().Count);
}
finally
{
IOUtils.Cleanup(null, stm, fileSystem);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestRecoverLease()
{
NUnit.Framework.Assert.AreEqual("ret: 1, You must supply a -path argument to recoverLease."
, RunCmd(new string[] { "recoverLease", "-retries", "1" }));
FSDataOutputStream @out = fs.Create(new Path("/foo"));
@out.Write(123);
@out.Close();
NUnit.Framework.Assert.AreEqual("ret: 0, recoverLease SUCCEEDED on /foo", RunCmd(
new string[] { "recoverLease", "-path", "/foo" }));
}
public virtual void TestFailedAppendBlockRejection()
{
Configuration conf = new HdfsConfiguration();
conf.Set("dfs.client.block.write.replace-datanode-on-failure.enable", "false");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
DistributedFileSystem fs = null;
try
{
fs = cluster.GetFileSystem();
Path path = new Path("/test");
FSDataOutputStream @out = fs.Create(path);
@out.WriteBytes("hello\n");
@out.Close();
// stop one datanode
MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0);
string dnAddress = dnProp.datanode.GetXferAddress().ToString();
if (dnAddress.StartsWith("/"))
{
dnAddress = Sharpen.Runtime.Substring(dnAddress, 1);
}
// append again to bump genstamps
for (int i = 0; i < 2; i++)
{
@out = fs.Append(path);
@out.WriteBytes("helloagain\n");
@out.Close();
}
// re-open and make the block state as underconstruction
@out = fs.Append(path);
cluster.RestartDataNode(dnProp, true);
// wait till the block report comes
Sharpen.Thread.Sleep(2000);
// check the block locations, this should not contain restarted datanode
BlockLocation[] locations = fs.GetFileBlockLocations(path, 0, long.MaxValue);
string[] names = locations[0].GetNames();
foreach (string node in names)
{
if (node.Equals(dnAddress))
{
NUnit.Framework.Assert.Fail("Failed append should not be present in latest block locations."
);
}
}
@out.Close();
}
finally
{
IOUtils.CloseStream(fs);
cluster.Shutdown();
}
}
/// <summary>Test to verify the race between finalizeBlock and Lease recovery</summary>
/// <exception cref="System.Exception"/>
public virtual void TestRaceBetweenReplicaRecoveryAndFinalizeBlock()
{
TearDown();
// Stop the Mocked DN started in startup()
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisKey, "1000");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
try
{
cluster.WaitClusterUp();
DistributedFileSystem fs = cluster.GetFileSystem();
Path path = new Path("/test");
FSDataOutputStream @out = fs.Create(path);
@out.WriteBytes("data");
@out.Hsync();
IList <LocatedBlock> blocks = DFSTestUtil.GetAllBlocks(fs.Open(path));
LocatedBlock block = blocks[0];
DataNode dataNode = cluster.GetDataNodes()[0];
AtomicBoolean recoveryInitResult = new AtomicBoolean(true);
Sharpen.Thread recoveryThread = new _Thread_612(block, dataNode, recoveryInitResult
);
recoveryThread.Start();
try
{
@out.Close();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue("Writing should fail", e.Message.Contains("are bad. Aborting..."
));
}
finally
{
recoveryThread.Join();
}
NUnit.Framework.Assert.IsTrue("Recovery should be initiated successfully", recoveryInitResult
.Get());
dataNode.UpdateReplicaUnderRecovery(block.GetBlock(), block.GetBlock().GetGenerationStamp
() + 1, block.GetBlock().GetBlockId(), block.GetBlockSize());
}
finally
{
if (null != cluster)
{
cluster.Shutdown();
cluster = null;
}
}
}
public virtual void TestImmediateRecoveryOfLease()
{
//create a file
// write bytes into the file.
byte[] actual = new byte[FileSize];
int size = AppendTestUtil.NextInt(FileSize);
Path filepath = CreateFile("/immediateRecoverLease-shortlease", size, true);
// set the soft limit to be 1 second so that the
// namenode triggers lease recovery on next attempt to write-for-open.
cluster.SetLeasePeriod(ShortLeasePeriod, LongLeasePeriod);
RecoverLeaseUsingCreate(filepath);
VerifyFile(dfs, filepath, actual, size);
//test recoverLease
// set the soft limit to be 1 hour but recoverLease should
// close the file immediately
cluster.SetLeasePeriod(LongLeasePeriod, LongLeasePeriod);
size = AppendTestUtil.NextInt(FileSize);
filepath = CreateFile("/immediateRecoverLease-longlease", size, false);
// test recoverLease from a different client
RecoverLease(filepath, null);
VerifyFile(dfs, filepath, actual, size);
// test recoverlease from the same client
size = AppendTestUtil.NextInt(FileSize);
filepath = CreateFile("/immediateRecoverLease-sameclient", size, false);
// create another file using the same client
Path filepath1 = new Path(filepath.ToString() + AppendTestUtil.NextInt());
FSDataOutputStream stm = dfs.Create(filepath1, true, BufSize, ReplicationNum, BlockSize
);
// recover the first file
RecoverLease(filepath, dfs);
VerifyFile(dfs, filepath, actual, size);
// continue to write to the second file
stm.Write(buffer, 0, size);
stm.Close();
VerifyFile(dfs, filepath1, actual, size);
}
/// <exception cref="System.IO.IOException"/>
private void DoWriteAndAbort(DistributedFileSystem fs, Path path)
{
fs.Mkdirs(path);
fs.AllowSnapshot(path);
DFSTestUtil.CreateFile(fs, new Path("/test/test1"), 100, (short)2, 100024L);
DFSTestUtil.CreateFile(fs, new Path("/test/test2"), 100, (short)2, 100024L);
Path file = new Path("/test/test/test2");
FSDataOutputStream @out = fs.Create(file);
for (int i = 0; i < 2; i++)
{
long count = 0;
while (count < 1048576)
{
@out.WriteBytes("hell");
count += 4;
}
}
((DFSOutputStream)@out.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
DFSTestUtil.AbortStream((DFSOutputStream)@out.GetWrappedStream());
Path file2 = new Path("/test/test/test3");
FSDataOutputStream out2 = fs.Create(file2);
for (int i_1 = 0; i_1 < 2; i_1++)
{
long count = 0;
while (count < 1048576)
{
out2.WriteBytes("hell");
count += 4;
}
}
((DFSOutputStream)out2.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.UpdateLength));
DFSTestUtil.AbortStream((DFSOutputStream)out2.GetWrappedStream());
fs.CreateSnapshot(path, "s1");
}
public virtual void TestAppend()
{
Configuration conf = new HdfsConfiguration();
short Replication = (short)3;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
try
{
DistributedFileSystem fs = cluster.GetFileSystem();
Path f = new Path(Dir, "testAppend");
{
Log.Info("create an empty file " + f);
fs.Create(f, Replication).Close();
FileStatus status = fs.GetFileStatus(f);
NUnit.Framework.Assert.AreEqual(Replication, status.GetReplication());
NUnit.Framework.Assert.AreEqual(0L, status.GetLen());
}
byte[] bytes = new byte[1000];
{
Log.Info("append " + bytes.Length + " bytes to " + f);
FSDataOutputStream @out = fs.Append(f);
@out.Write(bytes);
@out.Close();
FileStatus status = fs.GetFileStatus(f);
NUnit.Framework.Assert.AreEqual(Replication, status.GetReplication());
NUnit.Framework.Assert.AreEqual(bytes.Length, status.GetLen());
}
{
Log.Info("append another " + bytes.Length + " bytes to " + f);
try
{
FSDataOutputStream @out = fs.Append(f);
@out.Write(bytes);
@out.Close();
NUnit.Framework.Assert.Fail();
}
catch (IOException ioe)
{
Log.Info("This exception is expected", ioe);
}
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestTC1()
{
Path p = new Path("/TC1/foo");
System.Console.Out.WriteLine("p=" + p);
//a. Create file and write one block of data. Close file.
int len1 = (int)BlockSize;
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
// Reopen file to append. Append half block of data. Close file.
int len2 = (int)BlockSize / 2;
{
FSDataOutputStream @out = fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//b. Reopen file and read 1.5 blocks worth of data. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
}
/// <exception cref="System.Exception"/>
public virtual void TestTwoReplicaSameStorageTypeShouldNotSelect()
{
// HDFS-8147
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).StorageTypes
(new StorageType[][] { new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType[] { StorageType.Disk, StorageType.Disk }, new StorageType[]
{ StorageType.Disk, StorageType.Archive } }).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string file = "/testForTwoReplicaSameStorageTypeShouldNotSelect";
// write to DISK
FSDataOutputStream @out = dfs.Create(new Path(file), (short)2);
@out.WriteChars("testForTwoReplicaSameStorageTypeShouldNotSelect");
@out.Close();
// verify before movement
LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
StorageType[] storageTypes = lb.GetStorageTypes();
foreach (StorageType storageType in storageTypes)
{
NUnit.Framework.Assert.IsTrue(StorageType.Disk == storageType);
}
// move to ARCHIVE
dfs.SetStoragePolicy(new Path(file), "COLD");
int rc = ToolRunner.Run(conf, new Mover.Cli(), new string[] { "-p", file.ToString
() });
NUnit.Framework.Assert.AreEqual("Movement to ARCHIVE should be successfull", 0, rc
);
// Wait till namenode notified
Sharpen.Thread.Sleep(3000);
lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
storageTypes = lb.GetStorageTypes();
int archiveCount = 0;
foreach (StorageType storageType_1 in storageTypes)
{
if (StorageType.Archive == storageType_1)
{
archiveCount++;
}
}
NUnit.Framework.Assert.AreEqual(archiveCount, 2);
}
finally
{
cluster.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestFavoredNodesEndToEnd()
{
//create 10 files with random preferred nodes
for (int i = 0; i < NumFiles; i++)
{
Random rand = new Random(Runtime.CurrentTimeMillis() + i);
//pass a new created rand so as to get a uniform distribution each time
//without too much collisions (look at the do-while loop in getDatanodes)
IPEndPoint[] datanode = GetDatanodes(rand);
Path p = new Path("/filename" + i);
FSDataOutputStream @out = dfs.Create(p, FsPermission.GetDefault(), true, 4096, (short
)3, 4096L, null, datanode);
@out.Write(SomeBytes);
@out.Close();
BlockLocation[] locations = GetBlockLocations(p);
//verify the files got created in the right nodes
foreach (BlockLocation loc in locations)
{
string[] hosts = loc.GetNames();
string[] hosts1 = GetStringForInetSocketAddrs(datanode);
NUnit.Framework.Assert.IsTrue(CompareNodes(hosts, hosts1));
}
}
}
public virtual void TestAppend2AfterSoftLimit()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1);
//Set small soft-limit for lease
long softLimit = 1L;
long hardLimit = 9999999L;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.SetLeasePeriod(softLimit, hardLimit);
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
DistributedFileSystem fs2 = new DistributedFileSystem();
fs2.Initialize(fs.GetUri(), conf);
Path testPath = new Path("/testAppendAfterSoftLimit");
byte[] fileContents = AppendTestUtil.InitBuffer(32);
// create a new file without closing
FSDataOutputStream @out = fs.Create(testPath);
@out.Write(fileContents);
//Wait for > soft-limit
Sharpen.Thread.Sleep(250);
try
{
FSDataOutputStream appendStream2 = fs2.Append(testPath, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null);
appendStream2.Write(fileContents);
appendStream2.Close();
NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen
());
// make sure we now have 1 block since the first writer was revoked
LocatedBlocks blks = fs.GetClient().GetLocatedBlocks(testPath.ToString(), 0L);
NUnit.Framework.Assert.AreEqual(1, blks.GetLocatedBlocks().Count);
foreach (LocatedBlock blk in blks.GetLocatedBlocks())
{
NUnit.Framework.Assert.AreEqual(fileContents.Length, blk.GetBlockSize());
}
}
finally
{
fs.Close();
fs2.Close();
cluster.Shutdown();
}
}
public virtual void TestBlockRecoveryWithLessMetafile()
{
Configuration conf = new Configuration();
conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser
().GetShortUserName());
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
Path file = new Path("/testRecoveryFile");
DistributedFileSystem dfs = cluster.GetFileSystem();
FSDataOutputStream @out = dfs.Create(file);
int count = 0;
while (count < 2 * 1024 * 1024)
{
@out.WriteBytes("Data");
count += 4;
}
@out.Hsync();
// abort the original stream
((DFSOutputStream)@out.GetWrappedStream()).Abort();
LocatedBlocks locations = cluster.GetNameNodeRpc().GetBlockLocations(file.ToString
(), 0, count);
ExtendedBlock block = locations.Get(0).GetBlock();
DataNode dn = cluster.GetDataNodes()[0];
BlockLocalPathInfo localPathInfo = dn.GetBlockLocalPathInfo(block, null);
FilePath metafile = new FilePath(localPathInfo.GetMetaPath());
NUnit.Framework.Assert.IsTrue(metafile.Exists());
// reduce the block meta file size
RandomAccessFile raf = new RandomAccessFile(metafile, "rw");
raf.SetLength(metafile.Length() - 20);
raf.Close();
// restart DN to make replica to RWR
MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0);
cluster.RestartDataNode(dnProp, true);
// try to recover the lease
DistributedFileSystem newdfs = (DistributedFileSystem)FileSystem.NewInstance(cluster
.GetConfiguration(0));
count = 0;
while (++count < 10 && !newdfs.RecoverLease(file))
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue("File should be closed", newdfs.RecoverLease(file));
}