public virtual void TestTC2()
{
Path p = new Path("/TC2/foo");
System.Console.Out.WriteLine("p=" + p);
//a. Create file with one and a half block of data. Close file.
int len1 = (int)(BlockSize + BlockSize / 2);
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
AppendTestUtil.Check(fs, p, len1);
// Reopen file to append quarter block of data. Close file.
int len2 = (int)BlockSize / 4;
{
FSDataOutputStream @out = fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//b. Reopen file and read 1.75 blocks of data. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
}
public virtual void TestSmallAppendRace()
{
Path file = new Path("/testSmallAppendRace");
string fName = file.ToUri().GetPath();
// Create the file and write a small amount of data.
FSDataOutputStream stm = fs.Create(file);
AppendTestUtil.Write(stm, 0, 123);
stm.Close();
// Introduce a delay between getFileInfo and calling append() against NN.
DFSClient client = DFSClientAdapter.GetDFSClient(fs);
DFSClient spyClient = Org.Mockito.Mockito.Spy(client);
Org.Mockito.Mockito.When(spyClient.GetFileInfo(fName)).ThenAnswer(new _Answer_548
(client, fName));
DFSClientAdapter.SetDFSClient(fs, spyClient);
// Create two threads for doing appends to the same file.
Sharpen.Thread worker1 = new _Thread_564(this, file);
Sharpen.Thread worker2 = new _Thread_574(this, file);
worker1.Start();
worker2.Start();
// append will fail when the file size crosses the checksum chunk boundary,
// if append was called with a stale file stat.
DoSmallAppends(file, fs, 20);
}
/// <summary>TC12: Append to partial CRC chunk</summary>
/// <exception cref="System.Exception"/>
private void TestTC12(bool appendToNewBlock)
{
Path p = new Path("/TC12/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file with a block size of 64KB
// and a default io.bytes.per.checksum of 512 bytes.
// Write 25687 bytes of data. Close file.
int len1 = 25687;
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
//b. Reopen file in "append" mode. Append another 5877 bytes of data. Close file.
int len2 = 5877;
{
FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//c. Reopen file and read 25687+5877 bytes of data from file. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
if (appendToNewBlock)
{
LocatedBlocks blks = fs.dfs.GetLocatedBlocks(p.ToString(), 0);
NUnit.Framework.Assert.AreEqual(2, blks.GetLocatedBlocks().Count);
NUnit.Framework.Assert.AreEqual(len1, blks.GetLocatedBlocks()[0].GetBlockSize());
NUnit.Framework.Assert.AreEqual(len2, blks.GetLocatedBlocks()[1].GetBlockSize());
AppendTestUtil.Check(fs, p, 0, len1);
AppendTestUtil.Check(fs, p, len1, len2);
}
}
public virtual void TestTC2ForAppend2()
{
Path p = new Path("/TC2/foo2");
//a. Create file with one and a half block of data. Close file.
int len1 = (int)(BlockSize + BlockSize / 2);
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
AppendTestUtil.Check(fs, p, len1);
// Reopen file to append quarter block of data. Close file.
int len2 = (int)BlockSize / 4;
{
FSDataOutputStream @out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock
), 4096, null);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
// b. Reopen file and read 1.75 blocks of data. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
IList <LocatedBlock> blocks = fs.GetClient().GetLocatedBlocks(p.ToString(), 0L).GetLocatedBlocks
();
NUnit.Framework.Assert.AreEqual(3, blocks.Count);
NUnit.Framework.Assert.AreEqual(BlockSize, blocks[0].GetBlockSize());
NUnit.Framework.Assert.AreEqual(BlockSize / 2, blocks[1].GetBlockSize());
NUnit.Framework.Assert.AreEqual(BlockSize / 4, blocks[2].GetBlockSize());
}
public virtual void TestAppendWithPipelineRecovery()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
FSDataOutputStream @out = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(true).ManageNameDfsDirs
(true).NumDataNodes(4).Racks(new string[] { "/rack1", "/rack1", "/rack2", "/rack2" }).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
Path path = new Path("/test1");
@out = fs.Create(path, true, BlockSize, (short)3, BlockSize);
AppendTestUtil.Write(@out, 0, 1024);
@out.Close();
cluster.StopDataNode(3);
@out = fs.Append(path);
AppendTestUtil.Write(@out, 1024, 1024);
@out.Close();
cluster.RestartNameNode(true);
AppendTestUtil.Check(fs, path, 2048);
}
finally
{
IOUtils.CloseStream(@out);
if (null != cluster)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Test which randomly alternates between appending with
/// CRC32 and with CRC32C, crossing several block boundaries.
/// </summary>
/// <remarks>
/// Test which randomly alternates between appending with
/// CRC32 and with CRC32C, crossing several block boundaries.
/// Then, checks that all of the data can be read back correct.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestAlgoSwitchRandomized()
{
FileSystem fsWithCrc32 = CreateFsWithChecksum("CRC32", 512);
FileSystem fsWithCrc32C = CreateFsWithChecksum("CRC32C", 512);
Path p = new Path("/testAlgoSwitchRandomized");
long seed = Time.Now();
System.Console.Out.WriteLine("seed: " + seed);
Random r = new Random(seed);
// Create empty to start
IOUtils.CloseStream(fsWithCrc32.Create(p));
long st = Time.Now();
int len = 0;
while (Time.Now() - st < RandomTestRuntime)
{
int thisLen = r.Next(500);
FileSystem fs = (r.NextBoolean() ? fsWithCrc32 : fsWithCrc32C);
FSDataOutputStream stm = fs.Append(p);
try
{
AppendTestUtil.Write(stm, len, thisLen);
}
finally
{
stm.Close();
}
len += thisLen;
}
AppendTestUtil.Check(fsWithCrc32, p, len);
AppendTestUtil.Check(fsWithCrc32C, p, len);
}
/// <summary>
/// Test case that stops a writer after finalizing a block but
/// before calling completeFile, recovers a file from another writer,
/// starts writing from that writer, and then has the old lease holder
/// call completeFile
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCompleteOtherLeaseHoldersFile()
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Build();
try
{
cluster.WaitActive();
NamenodeProtocols preSpyNN = cluster.GetNameNodeRpc();
NamenodeProtocols spyNN = Org.Mockito.Mockito.Spy(preSpyNN);
// Delay completeFile
GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
Org.Mockito.Mockito.DoAnswer(delayer).When(spyNN).Complete(Matchers.AnyString(),
Matchers.AnyString(), (ExtendedBlock)Matchers.AnyObject(), Matchers.AnyLong());
DFSClient client = new DFSClient(null, spyNN, conf, null);
file1 = new Path("/testCompleteOtherLease");
OutputStream stm = client.Create("/testCompleteOtherLease", true);
// write 1/2 block
AppendTestUtil.Write(stm, 0, 4096);
AtomicReference <Exception> err = new AtomicReference <Exception>();
Sharpen.Thread t = new _Thread_242(stm, err);
t.Start();
Log.Info("Waiting for close to get to latch...");
delayer.WaitForCall();
// At this point, the block is finalized on the DNs, but the file
// has not been completed in the NN.
// Lose the leases
Log.Info("Killing lease checker");
client.GetLeaseRenewer().InterruptAndJoin();
FileSystem fs1 = cluster.GetFileSystem();
FileSystem fs2 = AppendTestUtil.CreateHdfsWithDifferentUsername(fs1.GetConf());
Log.Info("Recovering file");
RecoverFile(fs2);
Log.Info("Opening file for append from new fs");
FSDataOutputStream appenderStream = fs2.Append(file1);
Log.Info("Writing some data from new appender");
AppendTestUtil.Write(appenderStream, 0, 4096);
Log.Info("Telling old close to proceed.");
delayer.Proceed();
Log.Info("Waiting for close to finish.");
t.Join();
Log.Info("Close finished.");
// We expect that close will get a "Lease mismatch"
// error.
Exception thrownByClose = err.Get();
NUnit.Framework.Assert.IsNotNull(thrownByClose);
NUnit.Framework.Assert.IsTrue(thrownByClose is IOException);
if (!thrownByClose.Message.Contains("Lease mismatch"))
{
throw thrownByClose;
}
// The appender should be able to close properly
appenderStream.Close();
}
finally
{
cluster.Shutdown();
}
}
/// <summary>TC11: Racing rename</summary>
/// <exception cref="System.Exception"/>
private void TestTC11(bool appendToNewBlock)
{
Path p = new Path("/TC11/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file and write one block of data. Close file.
int len1 = (int)BlockSize;
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
//b. Reopen file in "append" mode. Append half block of data.
FSDataOutputStream out_1 = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.
Append, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
int len2 = (int)BlockSize / 2;
AppendTestUtil.Write(out_1, len1, len2);
out_1.Hflush();
//c. Rename file to file.new.
Path pnew = new Path(p + ".new");
NUnit.Framework.Assert.IsTrue(fs.Rename(p, pnew));
//d. Close file handle that was opened in (b).
out_1.Close();
//check block sizes
long len = fs.GetFileStatus(pnew).GetLen();
LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(pnew.ToString
(), 0L, len);
int numblock = locatedblocks.LocatedBlockCount();
for (int i = 0; i < numblock; i++)
{
LocatedBlock lb = locatedblocks.Get(i);
ExtendedBlock blk = lb.GetBlock();
long size = lb.GetBlockSize();
if (i < numblock - 1)
{
NUnit.Framework.Assert.AreEqual(BlockSize, size);
}
foreach (DatanodeInfo datanodeinfo in lb.GetLocations())
{
DataNode dn = cluster.GetDataNode(datanodeinfo.GetIpcPort());
Block metainfo = DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(blk.GetBlockPoolId
(), blk.GetBlockId());
NUnit.Framework.Assert.AreEqual(size, metainfo.GetNumBytes());
}
}
}
/// <summary>TC7: Corrupted replicas are present.</summary>
/// <exception cref="System.IO.IOException">an exception might be thrown</exception>
/// <exception cref="System.Exception"/>
private void TestTC7(bool appendToNewBlock)
{
short repl = 2;
Path p = new Path("/TC7/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file with replication factor of 2. Write half block of data. Close file.
int len1 = (int)(BlockSize / 2);
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, repl, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
DFSTestUtil.WaitReplication(fs, p, repl);
//b. Log into one datanode that has one replica of this block.
// Find the block file on this datanode and truncate it to zero size.
LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(p.ToString()
, 0L, len1);
NUnit.Framework.Assert.AreEqual(1, locatedblocks.LocatedBlockCount());
LocatedBlock lb = locatedblocks.Get(0);
ExtendedBlock blk = lb.GetBlock();
NUnit.Framework.Assert.AreEqual(len1, lb.GetBlockSize());
DatanodeInfo[] datanodeinfos = lb.GetLocations();
NUnit.Framework.Assert.AreEqual(repl, datanodeinfos.Length);
DataNode dn = cluster.GetDataNode(datanodeinfos[0].GetIpcPort());
FilePath f = DataNodeTestUtils.GetBlockFile(dn, blk.GetBlockPoolId(), blk.GetLocalBlock
());
RandomAccessFile raf = new RandomAccessFile(f, "rw");
AppendTestUtil.Log.Info("dn=" + dn + ", blk=" + blk + " (length=" + blk.GetNumBytes
() + ")");
NUnit.Framework.Assert.AreEqual(len1, raf.Length());
raf.SetLength(0);
raf.Close();
//c. Open file in "append mode". Append a new block worth of data. Close file.
int len2 = (int)BlockSize;
{
FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//d. Reopen file and read two blocks worth of data.
AppendTestUtil.Check(fs, p, len1 + len2);
}
// Do small appends.
/// <exception cref="System.IO.IOException"/>
internal virtual void DoSmallAppends(Path file, DistributedFileSystem fs, int iterations
)
{
for (int i = 0; i < iterations; i++)
{
FSDataOutputStream stm;
try
{
stm = fs.Append(file);
}
catch (IOException)
{
// If another thread is already appending, skip this time.
continue;
}
// Failure in write or close will be terminal.
AppendTestUtil.Write(stm, 0, 123);
stm.Close();
}
}
public virtual void TestTC1ForAppend2()
{
Path p = new Path("/TC1/foo2");
//a. Create file and write one block of data. Close file.
int len1 = (int)BlockSize;
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
// Reopen file to append. Append half block of data. Close file.
int len2 = (int)BlockSize / 2;
{
FSDataOutputStream @out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock
), 4096, null);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
// b. Reopen file and read 1.5 blocks worth of data. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
}
/// <exception cref="System.IO.IOException"/>
private void WriteAndAppend(FileSystem fs, Path p, int lengthForCreate, int lengthForAppend
)
{
// Creating a file with 4096 blockSize to write multiple blocks
FSDataOutputStream stream = fs.Create(p, true, BlockSize, (short)1, BlockSize);
try
{
AppendTestUtil.Write(stream, 0, lengthForCreate);
stream.Close();
stream = fs.Append(p);
AppendTestUtil.Write(stream, lengthForCreate, lengthForAppend);
stream.Close();
}
finally
{
IOUtils.CloseStream(stream);
}
int totalLength = lengthForCreate + lengthForAppend;
NUnit.Framework.Assert.AreEqual(totalLength, fs.GetFileStatus(p).GetLen());
}
/// <exception cref="System.IO.IOException"/>
private void AppendWithTwoFs(Path p, FileSystem fs1, FileSystem fs2)
{
FSDataOutputStream stm = fs1.Create(p);
try
{
AppendTestUtil.Write(stm, 0, SegmentLength);
}
finally
{
stm.Close();
}
stm = fs2.Append(p);
try
{
AppendTestUtil.Write(stm, SegmentLength, SegmentLength);
}
finally
{
stm.Close();
}
}
public virtual void TestTC5()
{
Path p = new Path("/TC5/foo");
System.Console.Out.WriteLine("p=" + p);
{
//a. Create file on Machine M1. Write half block to it. Close file.
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, (int)(BlockSize / 2));
@out.Close();
}
//b. Reopen file in "append" mode on Machine M1.
FSDataOutputStream out_1 = fs.Append(p);
//c. On Machine M2, reopen file in "append" mode. This should fail.
try
{
AppendTestUtil.CreateHdfsWithDifferentUsername(conf).Append(p);
NUnit.Framework.Assert.Fail("This should fail.");
}
catch (IOException ioe)
{
AppendTestUtil.Log.Info("GOOD: got an exception", ioe);
}
try
{
((DistributedFileSystem)AppendTestUtil.CreateHdfsWithDifferentUsername(conf)).Append
(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock), 4096, null);
NUnit.Framework.Assert.Fail("This should fail.");
}
catch (IOException ioe)
{
AppendTestUtil.Log.Info("GOOD: got an exception", ioe);
}
//d. On Machine M1, close file.
out_1.Close();
}
