public virtual void TestTC2() { Path p = new Path("/TC2/foo"); System.Console.Out.WriteLine("p=" + p); //a. Create file with one and a half block of data. Close file. int len1 = (int)(BlockSize + BlockSize / 2); { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } AppendTestUtil.Check(fs, p, len1); // Reopen file to append quarter block of data. Close file. int len2 = (int)BlockSize / 4; { FSDataOutputStream @out = fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //b. Reopen file and read 1.75 blocks of data. Close file. AppendTestUtil.Check(fs, p, len1 + len2); }
/// <summary>TC12: Append to partial CRC chunk</summary> /// <exception cref="System.Exception"/> private void TestTC12(bool appendToNewBlock) { Path p = new Path("/TC12/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file with a block size of 64KB // and a default io.bytes.per.checksum of 512 bytes. // Write 25687 bytes of data. Close file. int len1 = 25687; { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } //b. Reopen file in "append" mode. Append another 5877 bytes of data. Close file. int len2 = 5877; { FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null) : fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //c. Reopen file and read 25687+5877 bytes of data from file. Close file. AppendTestUtil.Check(fs, p, len1 + len2); if (appendToNewBlock) { LocatedBlocks blks = fs.dfs.GetLocatedBlocks(p.ToString(), 0); NUnit.Framework.Assert.AreEqual(2, blks.GetLocatedBlocks().Count); NUnit.Framework.Assert.AreEqual(len1, blks.GetLocatedBlocks()[0].GetBlockSize()); NUnit.Framework.Assert.AreEqual(len2, blks.GetLocatedBlocks()[1].GetBlockSize()); AppendTestUtil.Check(fs, p, 0, len1); AppendTestUtil.Check(fs, p, len1, len2); } }
public virtual void TestTC2ForAppend2() { Path p = new Path("/TC2/foo2"); //a. Create file with one and a half block of data. Close file. int len1 = (int)(BlockSize + BlockSize / 2); { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } AppendTestUtil.Check(fs, p, len1); // Reopen file to append quarter block of data. Close file. int len2 = (int)BlockSize / 4; { FSDataOutputStream @out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock ), 4096, null); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } // b. Reopen file and read 1.75 blocks of data. Close file. AppendTestUtil.Check(fs, p, len1 + len2); IList <LocatedBlock> blocks = fs.GetClient().GetLocatedBlocks(p.ToString(), 0L).GetLocatedBlocks (); NUnit.Framework.Assert.AreEqual(3, blocks.Count); NUnit.Framework.Assert.AreEqual(BlockSize, blocks[0].GetBlockSize()); NUnit.Framework.Assert.AreEqual(BlockSize / 2, blocks[1].GetBlockSize()); NUnit.Framework.Assert.AreEqual(BlockSize / 4, blocks[2].GetBlockSize()); }
public virtual void TestAppendWithPipelineRecovery() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; FSDataOutputStream @out = null; try { cluster = new MiniDFSCluster.Builder(conf).ManageDataDfsDirs(true).ManageNameDfsDirs (true).NumDataNodes(4).Racks(new string[] { "/rack1", "/rack1", "/rack2", "/rack2" }).Build(); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); Path path = new Path("/test1"); @out = fs.Create(path, true, BlockSize, (short)3, BlockSize); AppendTestUtil.Write(@out, 0, 1024); @out.Close(); cluster.StopDataNode(3); @out = fs.Append(path); AppendTestUtil.Write(@out, 1024, 1024); @out.Close(); cluster.RestartNameNode(true); AppendTestUtil.Check(fs, path, 2048); } finally { IOUtils.CloseStream(@out); if (null != cluster) { cluster.Shutdown(); } } }
/// <summary> /// Test which randomly alternates between appending with /// CRC32 and with CRC32C, crossing several block boundaries. /// </summary> /// <remarks> /// Test which randomly alternates between appending with /// CRC32 and with CRC32C, crossing several block boundaries. /// Then, checks that all of the data can be read back correct. /// </remarks> /// <exception cref="System.IO.IOException"/> public virtual void TestAlgoSwitchRandomized() { FileSystem fsWithCrc32 = CreateFsWithChecksum("CRC32", 512); FileSystem fsWithCrc32C = CreateFsWithChecksum("CRC32C", 512); Path p = new Path("/testAlgoSwitchRandomized"); long seed = Time.Now(); System.Console.Out.WriteLine("seed: " + seed); Random r = new Random(seed); // Create empty to start IOUtils.CloseStream(fsWithCrc32.Create(p)); long st = Time.Now(); int len = 0; while (Time.Now() - st < RandomTestRuntime) { int thisLen = r.Next(500); FileSystem fs = (r.NextBoolean() ? fsWithCrc32 : fsWithCrc32C); FSDataOutputStream stm = fs.Append(p); try { AppendTestUtil.Write(stm, len, thisLen); } finally { stm.Close(); } len += thisLen; } AppendTestUtil.Check(fsWithCrc32, p, len); AppendTestUtil.Check(fsWithCrc32C, p, len); }
public virtual void TestSwitchChunkSize() { FileSystem fsWithSmallChunk = CreateFsWithChecksum("CRC32", 512); FileSystem fsWithBigChunk = CreateFsWithChecksum("CRC32", 1024); Path p = new Path("/testSwitchChunkSize"); AppendWithTwoFs(p, fsWithSmallChunk, fsWithBigChunk); AppendTestUtil.Check(fsWithSmallChunk, p, SegmentLength * 2); AppendTestUtil.Check(fsWithBigChunk, p, SegmentLength * 2); }
public virtual void TestSwitchAlgorithms() { FileSystem fsWithCrc32 = CreateFsWithChecksum("CRC32", 512); FileSystem fsWithCrc32C = CreateFsWithChecksum("CRC32C", 512); Path p = new Path("/testSwitchAlgorithms"); AppendWithTwoFs(p, fsWithCrc32, fsWithCrc32C); // Regardless of which FS is used to read, it should pick up // the on-disk checksum! AppendTestUtil.Check(fsWithCrc32C, p, SegmentLength * 2); AppendTestUtil.Check(fsWithCrc32, p, SegmentLength * 2); }
/// <summary>TC7: Corrupted replicas are present.</summary> /// <exception cref="System.IO.IOException">an exception might be thrown</exception> /// <exception cref="System.Exception"/> private void TestTC7(bool appendToNewBlock) { short repl = 2; Path p = new Path("/TC7/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file with replication factor of 2. Write half block of data. Close file. int len1 = (int)(BlockSize / 2); { FSDataOutputStream @out = fs.Create(p, false, buffersize, repl, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } DFSTestUtil.WaitReplication(fs, p, repl); //b. Log into one datanode that has one replica of this block. // Find the block file on this datanode and truncate it to zero size. LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(p.ToString() , 0L, len1); NUnit.Framework.Assert.AreEqual(1, locatedblocks.LocatedBlockCount()); LocatedBlock lb = locatedblocks.Get(0); ExtendedBlock blk = lb.GetBlock(); NUnit.Framework.Assert.AreEqual(len1, lb.GetBlockSize()); DatanodeInfo[] datanodeinfos = lb.GetLocations(); NUnit.Framework.Assert.AreEqual(repl, datanodeinfos.Length); DataNode dn = cluster.GetDataNode(datanodeinfos[0].GetIpcPort()); FilePath f = DataNodeTestUtils.GetBlockFile(dn, blk.GetBlockPoolId(), blk.GetLocalBlock ()); RandomAccessFile raf = new RandomAccessFile(f, "rw"); AppendTestUtil.Log.Info("dn=" + dn + ", blk=" + blk + " (length=" + blk.GetNumBytes () + ")"); NUnit.Framework.Assert.AreEqual(len1, raf.Length()); raf.SetLength(0); raf.Close(); //c. Open file in "append mode". Append a new block worth of data. Close file. int len2 = (int)BlockSize; { FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null) : fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //d. Reopen file and read two blocks worth of data. AppendTestUtil.Check(fs, p, len1 + len2); }
public virtual void TestTC1ForAppend2() { Path p = new Path("/TC1/foo2"); //a. Create file and write one block of data. Close file. int len1 = (int)BlockSize; { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } // Reopen file to append. Append half block of data. Close file. int len2 = (int)BlockSize / 2; { FSDataOutputStream @out = fs.Append(p, EnumSet.Of(CreateFlag.Append, CreateFlag.NewBlock ), 4096, null); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } // b. Reopen file and read 1.5 blocks worth of data. Close file. AppendTestUtil.Check(fs, p, len1 + len2); }
public virtual void TestAppendRestart() { Configuration conf = new HdfsConfiguration(); // Turn off persistent IPC, so that the DFSClient can survive NN restart conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0); MiniDFSCluster cluster = null; FSDataOutputStream stream = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); FileSystem fs = cluster.GetFileSystem(); FilePath editLog = new FilePath(FSImageTestUtil.GetNameNodeCurrentDirs(cluster, 0 )[0], NNStorage.GetInProgressEditsFileName(1)); EnumMap <FSEditLogOpCodes, Holder <int> > counts; Path p1 = new Path("/block-boundaries"); WriteAndAppend(fs, p1, BlockSize, BlockSize); counts = FSImageTestUtil.CountEditLogOpTypes(editLog); // OP_ADD to create file // OP_ADD_BLOCK for first block // OP_CLOSE to close file // OP_APPEND to reopen file // OP_ADD_BLOCK for second block // OP_CLOSE to close file NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpAdd].held); NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpAppend].held); NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAddBlock].held); NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpClose].held); Path p2 = new Path("/not-block-boundaries"); WriteAndAppend(fs, p2, BlockSize / 2, BlockSize); counts = FSImageTestUtil.CountEditLogOpTypes(editLog); // OP_ADD to create file // OP_ADD_BLOCK for first block // OP_CLOSE to close file // OP_APPEND to re-establish the lease // OP_UPDATE_BLOCKS from the updatePipeline call (increments genstamp of last block) // OP_ADD_BLOCK at the start of the second block // OP_CLOSE to close file // Total: 2 OP_ADDs, 1 OP_UPDATE_BLOCKS, 2 OP_ADD_BLOCKs, and 2 OP_CLOSEs // in addition to the ones above NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAdd].held); NUnit.Framework.Assert.AreEqual(2, (int)counts[FSEditLogOpCodes.OpAppend].held); NUnit.Framework.Assert.AreEqual(1, (int)counts[FSEditLogOpCodes.OpUpdateBlocks].held ); NUnit.Framework.Assert.AreEqual(2 + 2, (int)counts[FSEditLogOpCodes.OpAddBlock].held ); NUnit.Framework.Assert.AreEqual(2 + 2, (int)counts[FSEditLogOpCodes.OpClose].held ); cluster.RestartNameNode(); AppendTestUtil.Check(fs, p1, 2 * BlockSize); AppendTestUtil.Check(fs, p2, 3 * BlockSize / 2); } finally { IOUtils.CloseStream(stream); if (cluster != null) { cluster.Shutdown(); } } }