public virtual void TestTC2()
{
Path p = new Path("/TC2/foo");
System.Console.Out.WriteLine("p=" + p);
//a. Create file with one and a half block of data. Close file.
int len1 = (int)(BlockSize + BlockSize / 2);
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
AppendTestUtil.Check(fs, p, len1);
// Reopen file to append quarter block of data. Close file.
int len2 = (int)BlockSize / 4;
{
FSDataOutputStream @out = fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//b. Reopen file and read 1.75 blocks of data. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
}
/// <summary>TC12: Append to partial CRC chunk</summary>
/// <exception cref="System.Exception"/>
private void TestTC12(bool appendToNewBlock)
{
Path p = new Path("/TC12/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file with a block size of 64KB
// and a default io.bytes.per.checksum of 512 bytes.
// Write 25687 bytes of data. Close file.
int len1 = 25687;
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
//b. Reopen file in "append" mode. Append another 5877 bytes of data. Close file.
int len2 = 5877;
{
FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//c. Reopen file and read 25687+5877 bytes of data from file. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
if (appendToNewBlock)
{
LocatedBlocks blks = fs.dfs.GetLocatedBlocks(p.ToString(), 0);
NUnit.Framework.Assert.AreEqual(2, blks.GetLocatedBlocks().Count);
NUnit.Framework.Assert.AreEqual(len1, blks.GetLocatedBlocks()[0].GetBlockSize());
NUnit.Framework.Assert.AreEqual(len2, blks.GetLocatedBlocks()[1].GetBlockSize());
AppendTestUtil.Check(fs, p, 0, len1);
AppendTestUtil.Check(fs, p, len1, len2);
}
}
public virtual void TestTC2ForAppend2()
{
Path p = new Path("/TC2/foo2");
//a. Create file with one and a half block of data. Close file.
int len1 = (int)(BlockSize + BlockSize / 2);
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
AppendTestUtil.Check(fs, p, len1);
// Reopen file to append quarter block of data. Close file.
int len2 = (int)BlockSize / 4;
{
FSDataOutputStream @out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock
), 4096, null);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
// b. Reopen file and read 1.75 blocks of data. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
IList <LocatedBlock> blocks = fs.GetClient().GetLocatedBlocks(p.ToString(), 0L).GetLocatedBlocks
();
NUnit.Framework.Assert.AreEqual(3, blocks.Count);
NUnit.Framework.Assert.AreEqual(BlockSize, blocks[0].GetBlockSize());
NUnit.Framework.Assert.AreEqual(BlockSize / 2, blocks[1].GetBlockSize());
NUnit.Framework.Assert.AreEqual(BlockSize / 4, blocks[2].GetBlockSize());
}
public virtual void TestAppendWithPipelineRecovery()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
FSDataOutputStream @out = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(true).ManageNameDfsDirs
(true).NumDataNodes(4).Racks(new string[] { "/rack1", "/rack1", "/rack2", "/rack2" }).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
Path path = new Path("/test1");
@out = fs.Create(path, true, BlockSize, (short)3, BlockSize);
AppendTestUtil.Write(@out, 0, 1024);
@out.Close();
cluster.StopDataNode(3);
@out = fs.Append(path);
AppendTestUtil.Write(@out, 1024, 1024);
@out.Close();
cluster.RestartNameNode(true);
AppendTestUtil.Check(fs, path, 2048);
}
finally
{
IOUtils.CloseStream(@out);
if (null != cluster)
{
cluster.Shutdown();
}
}
}
/// <summary>
/// Test which randomly alternates between appending with
/// CRC32 and with CRC32C, crossing several block boundaries.
/// </summary>
/// <remarks>
/// Test which randomly alternates between appending with
/// CRC32 and with CRC32C, crossing several block boundaries.
/// Then, checks that all of the data can be read back correct.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestAlgoSwitchRandomized()
{
FileSystem fsWithCrc32 = CreateFsWithChecksum("CRC32", 512);
FileSystem fsWithCrc32C = CreateFsWithChecksum("CRC32C", 512);
Path p = new Path("/testAlgoSwitchRandomized");
long seed = Time.Now();
System.Console.Out.WriteLine("seed: " + seed);
Random r = new Random(seed);
// Create empty to start
IOUtils.CloseStream(fsWithCrc32.Create(p));
long st = Time.Now();
int len = 0;
while (Time.Now() - st < RandomTestRuntime)
{
int thisLen = r.Next(500);
FileSystem fs = (r.NextBoolean() ? fsWithCrc32 : fsWithCrc32C);
FSDataOutputStream stm = fs.Append(p);
try
{
AppendTestUtil.Write(stm, len, thisLen);
}
finally
{
stm.Close();
}
len += thisLen;
}
AppendTestUtil.Check(fsWithCrc32, p, len);
AppendTestUtil.Check(fsWithCrc32C, p, len);
}
public virtual void TestSwitchChunkSize()
{
FileSystem fsWithSmallChunk = CreateFsWithChecksum("CRC32", 512);
FileSystem fsWithBigChunk = CreateFsWithChecksum("CRC32", 1024);
Path p = new Path("/testSwitchChunkSize");
AppendWithTwoFs(p, fsWithSmallChunk, fsWithBigChunk);
AppendTestUtil.Check(fsWithSmallChunk, p, SegmentLength * 2);
AppendTestUtil.Check(fsWithBigChunk, p, SegmentLength * 2);
}
public virtual void TestSwitchAlgorithms()
{
FileSystem fsWithCrc32 = CreateFsWithChecksum("CRC32", 512);
FileSystem fsWithCrc32C = CreateFsWithChecksum("CRC32C", 512);
Path p = new Path("/testSwitchAlgorithms");
AppendWithTwoFs(p, fsWithCrc32, fsWithCrc32C);
// Regardless of which FS is used to read, it should pick up
// the on-disk checksum!
AppendTestUtil.Check(fsWithCrc32C, p, SegmentLength * 2);
AppendTestUtil.Check(fsWithCrc32, p, SegmentLength * 2);
}
/// <summary>TC7: Corrupted replicas are present.</summary>
/// <exception cref="System.IO.IOException">an exception might be thrown</exception>
/// <exception cref="System.Exception"/>
private void TestTC7(bool appendToNewBlock)
{
short repl = 2;
Path p = new Path("/TC7/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file with replication factor of 2. Write half block of data. Close file.
int len1 = (int)(BlockSize / 2);
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, repl, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
DFSTestUtil.WaitReplication(fs, p, repl);
//b. Log into one datanode that has one replica of this block.
// Find the block file on this datanode and truncate it to zero size.
LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(p.ToString()
, 0L, len1);
NUnit.Framework.Assert.AreEqual(1, locatedblocks.LocatedBlockCount());
LocatedBlock lb = locatedblocks.Get(0);
ExtendedBlock blk = lb.GetBlock();
NUnit.Framework.Assert.AreEqual(len1, lb.GetBlockSize());
DatanodeInfo[] datanodeinfos = lb.GetLocations();
NUnit.Framework.Assert.AreEqual(repl, datanodeinfos.Length);
DataNode dn = cluster.GetDataNode(datanodeinfos[0].GetIpcPort());
FilePath f = DataNodeTestUtils.GetBlockFile(dn, blk.GetBlockPoolId(), blk.GetLocalBlock
());
RandomAccessFile raf = new RandomAccessFile(f, "rw");
AppendTestUtil.Log.Info("dn=" + dn + ", blk=" + blk + " (length=" + blk.GetNumBytes
() + ")");
NUnit.Framework.Assert.AreEqual(len1, raf.Length());
raf.SetLength(0);
raf.Close();
//c. Open file in "append mode". Append a new block worth of data. Close file.
int len2 = (int)BlockSize;
{
FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//d. Reopen file and read two blocks worth of data.
AppendTestUtil.Check(fs, p, len1 + len2);
}
public virtual void TestTC1ForAppend2()
{
Path p = new Path("/TC1/foo2");
//a. Create file and write one block of data. Close file.
int len1 = (int)BlockSize;
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
// Reopen file to append. Append half block of data. Close file.
int len2 = (int)BlockSize / 2;
{
FSDataOutputStream @out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock
), 4096, null);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
// b. Reopen file and read 1.5 blocks worth of data. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
}
public virtual void TestAppendRestart()
{
Configuration conf = new HdfsConfiguration();
// Turn off persistent IPC, so that the DFSClient can survive NN restart
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
MiniDFSCluster cluster = null;
FSDataOutputStream stream = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
FileSystem fs = cluster.GetFileSystem();
FilePath editLog = new FilePath(FSImageTestUtil.GetNameNodeCurrentDirs(cluster, 0
)[0], NNStorage.GetInProgressEditsFileName(1));
EnumMap <FSEditLogOpCodes, Holder <int> > counts;
Path p1 = new Path("/block-boundaries");
WriteAndAppend(fs, p1, BlockSize, BlockSize);
counts = FSImageTestUtil.CountEditLogOpTypes(editLog);
// OP_ADD to create file
// OP_ADD_BLOCK for first block
// OP_CLOSE to close file
// OP_APPEND to reopen file
// OP_ADD_BLOCK for second block
// OP_CLOSE to close file
NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpAdd].held);
NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpAppend].held);
NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAddBlock].held);
NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpClose].held);
Path p2 = new Path("/not-block-boundaries");
WriteAndAppend(fs, p2, BlockSize / 2, BlockSize);
counts = FSImageTestUtil.CountEditLogOpTypes(editLog);
// OP_ADD to create file
// OP_ADD_BLOCK for first block
// OP_CLOSE to close file
// OP_APPEND to re-establish the lease
// OP_UPDATE_BLOCKS from the updatePipeline call (increments genstamp of last block)
// OP_ADD_BLOCK at the start of the second block
// OP_CLOSE to close file
// Total: 2 OP_ADDs, 1 OP_UPDATE_BLOCKS, 2 OP_ADD_BLOCKs, and 2 OP_CLOSEs
// in addition to the ones above
NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAdd].held);
NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAppend].held);
NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpUpdateBlocks].held
);
NUnit.Framework.Assert.AreEqual(2 + 2, (int)counts[FSEditLogOpCodes.OpAddBlock].held
);
NUnit.Framework.Assert.AreEqual(2 + 2, (int)counts[FSEditLogOpCodes.OpClose].held
);
cluster.RestartNameNode();
AppendTestUtil.Check(fs, p1, 2 * BlockSize);
AppendTestUtil.Check(fs, p2, 3 * BlockSize / 2);
}
finally
{
IOUtils.CloseStream(stream);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
