public virtual void TestCommitWithInvalidGenStamp()
{
Path file = new Path("/file");
FSDataOutputStream @out = null;
try
{
@out = dfs.Create(file, (short)1);
INodeFile fileNode = dir.GetINode4Write(file.ToString()).AsFile();
ExtendedBlock previous = null;
Block newBlock = DFSTestUtil.AddBlockToFile(cluster.GetDataNodes(), dfs, cluster.
GetNamesystem(), file.ToString(), fileNode, dfs.GetClient().GetClientName(), previous
, 100);
Block newBlockClone = new Block(newBlock);
previous = new ExtendedBlock(cluster.GetNamesystem().GetBlockPoolId(), newBlockClone
);
previous.SetGenerationStamp(123);
try
{
dfs.GetClient().GetNamenode().Complete(file.ToString(), dfs.GetClient().GetClientName
(), previous, fileNode.GetId());
NUnit.Framework.Assert.Fail("should throw exception because invalid genStamp");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.ToString().Contains("Commit block with mismatching GS. NN has "
+ newBlock + ", client submits " + newBlockClone));
}
previous = new ExtendedBlock(cluster.GetNamesystem().GetBlockPoolId(), newBlock);
bool complete = dfs.GetClient().GetNamenode().Complete(file.ToString(), dfs.GetClient
().GetClientName(), previous, fileNode.GetId());
NUnit.Framework.Assert.IsTrue("should complete successfully", complete);
}
finally
{
IOUtils.Cleanup(null, @out);
}
}
public virtual void TestOpWrite()
{
int numDataNodes = 1;
long BlockIdFudge = 128;
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
datanode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes()[0], poolId
);
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
/* Test writing to finalized replicas */
Path file = new Path("dataprotocol.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
// test PIPELINE_SETUP_CREATE on a finalized block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create an existing block"
, true);
// test PIPELINE_DATA_STREAMING on a finalized block
TestWrite(firstBlock, BlockConstructionStage.DataStreaming, 0L, "Unexpected stage"
, true);
// test PIPELINE_SETUP_STREAMING_RECOVERY on an existing block
long newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Cannot recover data streaming to a finalized replica", true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Append to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_APPEND_RECOVERY on an existing block
file = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover appending to a finalized replica", false);
// test PIPELINE_CLOSE_RECOVERY on an existing block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineCloseRecovery, newGS, "Recover failed close to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// Test writing to a new block. Don't choose the next sequential
// block ID to avoid conflicting with IDs chosen by the NN.
long newBlockId = firstBlock.GetBlockId() + BlockIdFudge;
ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId
, 0, firstBlock.GetGenerationStamp());
// test PIPELINE_SETUP_CREATE on a new block
TestWrite(newBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Create a new block"
, false);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a new block
newGS = newBlock.GetGenerationStamp() + 1;
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
TestWrite(newBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS,
"Recover a new block", true);
// test PIPELINE_SETUP_APPEND on a new block
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a new block"
, true);
// test PIPELINE_SETUP_APPEND_RECOVERY on a new block
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Cannot append to a new block"
, true);
/* Test writing to RBW replicas */
Path file1 = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file1, 1L, (short)numDataNodes, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file1).GetWrappedStream()
);
@out.Write(1);
@out.Hflush();
FSDataInputStream @in = fileSys.Open(file1);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
try
{
// test PIPELINE_SETUP_CREATE on a RBW block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create a RBW block"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a RBW replica"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover append to a RBW replica", false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a RBW block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
@in = fileSys.Open(file);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Recover a RBW replica", false);
}
finally
{
IOUtils.CloseStream(@in);
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
/// <summary>Constructor</summary>
/// <param name="block">Block that is being read</param>
/// <param name="startOffset">starting offset to read from</param>
/// <param name="length">length of data to read</param>
/// <param name="corruptChecksumOk">if true, corrupt checksum is okay</param>
/// <param name="verifyChecksum">verify checksum while reading the data</param>
/// <param name="sendChecksum">send checksum to client.</param>
/// <param name="datanode">datanode from which the block is being read</param>
/// <param name="clientTraceFmt">format string used to print client trace logs</param>
/// <exception cref="System.IO.IOException"/>
internal BlockSender(ExtendedBlock block, long startOffset, long length, bool corruptChecksumOk
, bool verifyChecksum, bool sendChecksum, DataNode datanode, string clientTraceFmt
, CachingStrategy cachingStrategy)
{
// Cache-management related fields
// 1MB
try
{
this.block = block;
this.corruptChecksumOk = corruptChecksumOk;
this.verifyChecksum = verifyChecksum;
this.clientTraceFmt = clientTraceFmt;
/*
* If the client asked for the cache to be dropped behind all reads,
* we honor that. Otherwise, we use the DataNode defaults.
* When using DataNode defaults, we use a heuristic where we only
* drop the cache for large reads.
*/
if (cachingStrategy.GetDropBehind() == null)
{
this.dropCacheBehindAllReads = false;
this.dropCacheBehindLargeReads = datanode.GetDnConf().dropCacheBehindReads;
}
else
{
this.dropCacheBehindAllReads = this.dropCacheBehindLargeReads = cachingStrategy.GetDropBehind
();
}
/*
* Similarly, if readahead was explicitly requested, we always do it.
* Otherwise, we read ahead based on the DataNode settings, and only
* when the reads are large.
*/
if (cachingStrategy.GetReadahead() == null)
{
this.alwaysReadahead = false;
this.readaheadLength = datanode.GetDnConf().readaheadLength;
}
else
{
this.alwaysReadahead = true;
this.readaheadLength = cachingStrategy.GetReadahead();
}
this.datanode = datanode;
if (verifyChecksum)
{
// To simplify implementation, callers may not specify verification
// without sending.
Preconditions.CheckArgument(sendChecksum, "If verifying checksum, currently must also send it."
);
}
Replica replica;
long replicaVisibleLength;
lock (datanode.data)
{
replica = GetReplica(block, datanode);
replicaVisibleLength = replica.GetVisibleLength();
}
// if there is a write in progress
ChunkChecksum chunkChecksum = null;
if (replica is ReplicaBeingWritten)
{
ReplicaBeingWritten rbw = (ReplicaBeingWritten)replica;
WaitForMinLength(rbw, startOffset + length);
chunkChecksum = rbw.GetLastChecksumAndDataLen();
}
if (replica.GetGenerationStamp() < block.GetGenerationStamp())
{
throw new IOException("Replica gen stamp < block genstamp, block=" + block + ", replica="
+ replica);
}
else
{
if (replica.GetGenerationStamp() > block.GetGenerationStamp())
{
if (DataNode.Log.IsDebugEnabled())
{
DataNode.Log.Debug("Bumping up the client provided" + " block's genstamp to latest "
+ replica.GetGenerationStamp() + " for block " + block);
}
block.SetGenerationStamp(replica.GetGenerationStamp());
}
}
if (replicaVisibleLength < 0)
{
throw new IOException("Replica is not readable, block=" + block + ", replica=" +
replica);
}
if (DataNode.Log.IsDebugEnabled())
{
DataNode.Log.Debug("block=" + block + ", replica=" + replica);
}
// transferToFully() fails on 32 bit platforms for block sizes >= 2GB,
// use normal transfer in those cases
this.transferToAllowed = datanode.GetDnConf().transferToAllowed&& (!is32Bit || length
<= int.MaxValue);
// Obtain a reference before reading data
this.volumeRef = datanode.data.GetVolume(block).ObtainReference();
/*
* (corruptChecksumOK, meta_file_exist): operation
* True, True: will verify checksum
* True, False: No verify, e.g., need to read data from a corrupted file
* False, True: will verify checksum
* False, False: throws IOException file not found
*/
DataChecksum csum = null;
if (verifyChecksum || sendChecksum)
{
LengthInputStream metaIn = null;
bool keepMetaInOpen = false;
try
{
metaIn = datanode.data.GetMetaDataInputStream(block);
if (!corruptChecksumOk || metaIn != null)
{
if (metaIn == null)
{
//need checksum but meta-data not found
throw new FileNotFoundException("Meta-data not found for " + block);
}
// The meta file will contain only the header if the NULL checksum
// type was used, or if the replica was written to transient storage.
// Checksum verification is not performed for replicas on transient
// storage. The header is important for determining the checksum
// type later when lazy persistence copies the block to non-transient
// storage and computes the checksum.
if (metaIn.GetLength() > BlockMetadataHeader.GetHeaderSize())
{
checksumIn = new DataInputStream(new BufferedInputStream(metaIn, HdfsConstants.IoFileBufferSize
));
csum = BlockMetadataHeader.ReadDataChecksum(checksumIn, block);
keepMetaInOpen = true;
}
}
else
{
Log.Warn("Could not find metadata file for " + block);
}
}
finally
{
if (!keepMetaInOpen)
{
IOUtils.CloseStream(metaIn);
}
}
}
if (csum == null)
{
// The number of bytes per checksum here determines the alignment
// of reads: we always start reading at a checksum chunk boundary,
// even if the checksum type is NULL. So, choosing too big of a value
// would risk sending too much unnecessary data. 512 (1 disk sector)
// is likely to result in minimal extra IO.
csum = DataChecksum.NewDataChecksum(DataChecksum.Type.Null, 512);
}
/*
* If chunkSize is very large, then the metadata file is mostly
* corrupted. For now just truncate bytesPerchecksum to blockLength.
*/
int size = csum.GetBytesPerChecksum();
if (size > 10 * 1024 * 1024 && size > replicaVisibleLength)
{
csum = DataChecksum.NewDataChecksum(csum.GetChecksumType(), Math.Max((int)replicaVisibleLength
, 10 * 1024 * 1024));
size = csum.GetBytesPerChecksum();
}
chunkSize = size;
checksum = csum;
checksumSize = checksum.GetChecksumSize();
length = length < 0 ? replicaVisibleLength : length;
// end is either last byte on disk or the length for which we have a
// checksum
long end = chunkChecksum != null?chunkChecksum.GetDataLength() : replica.GetBytesOnDisk
();
if (startOffset < 0 || startOffset > end || (length + startOffset) > end)
{
string msg = " Offset " + startOffset + " and length " + length + " don't match block "
+ block + " ( blockLen " + end + " )";
Log.Warn(datanode.GetDNRegistrationForBP(block.GetBlockPoolId()) + ":sendBlock() : "
+ msg);
throw new IOException(msg);
}
// Ensure read offset is position at the beginning of chunk
offset = startOffset - (startOffset % chunkSize);
if (length >= 0)
{
// Ensure endOffset points to end of chunk.
long tmpLen = startOffset + length;
if (tmpLen % chunkSize != 0)
{
tmpLen += (chunkSize - tmpLen % chunkSize);
}
if (tmpLen < end)
{
// will use on-disk checksum here since the end is a stable chunk
end = tmpLen;
}
else
{
if (chunkChecksum != null)
{
// last chunk is changing. flag that we need to use in-memory checksum
this.lastChunkChecksum = chunkChecksum;
}
}
}
endOffset = end;
// seek to the right offsets
if (offset > 0 && checksumIn != null)
{
long checksumSkip = (offset / chunkSize) * checksumSize;
// note blockInStream is seeked when created below
if (checksumSkip > 0)
{
// Should we use seek() for checksum file as well?
IOUtils.SkipFully(checksumIn, checksumSkip);
}
}
seqno = 0;
if (DataNode.Log.IsDebugEnabled())
{
DataNode.Log.Debug("replica=" + replica);
}
blockIn = datanode.data.GetBlockInputStream(block, offset);
// seek to offset
if (blockIn is FileInputStream)
{
blockInFd = ((FileInputStream)blockIn).GetFD();
}
else
{
blockInFd = null;
}
}
catch (IOException ioe)
{
IOUtils.CloseStream(this);
IOUtils.CloseStream(blockIn);
throw;
}
}
