private void UpdateDeletedBlockId(ExtendedBlock block)
{
lock (this)
{
ICollection <long> blockIds = deletedBlockIds[block.GetBlockPoolId()];
if (blockIds == null)
{
blockIds = new HashSet <long>();
deletedBlockIds[block.GetBlockPoolId()] = blockIds;
}
blockIds.AddItem(block.GetBlockId());
numDeletedBlocks++;
if (numDeletedBlocks == MaxDeletedBlocks)
{
foreach (KeyValuePair <string, ICollection <long> > e in deletedBlockIds)
{
string bpid = e.Key;
ICollection <long> bs = e.Value;
fsdatasetImpl.RemoveDeletedBlocks(bpid, bs);
bs.Clear();
}
numDeletedBlocks = 0;
}
}
}
/// <summary>Check if access should be allowed.</summary>
/// <remarks>
/// Check if access should be allowed. userID is not checked if null. This
/// method doesn't check if token password is correct. It should be used only
/// when token password has already been verified (e.g., in the RPC layer).
/// </remarks>
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/>
public virtual void CheckAccess(BlockTokenIdentifier id, string userId, ExtendedBlock
block, BlockTokenSecretManager.AccessMode mode)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Checking access for user="******", block=" + block + ", access mode="
+ mode + " using " + id.ToString());
}
if (userId != null && !userId.Equals(id.GetUserId()))
{
throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " doesn't belong to user "
+ userId);
}
if (!id.GetBlockPoolId().Equals(block.GetBlockPoolId()))
{
throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " doesn't apply to block "
+ block);
}
if (id.GetBlockId() != block.GetBlockId())
{
throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " doesn't apply to block "
+ block);
}
if (IsExpired(id.GetExpiryDate()))
{
throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " is expired."
);
}
if (!id.GetAccessModes().Contains(mode))
{
throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " doesn't have "
+ mode + " permission");
}
}
/// <exception cref="System.IO.IOException"/>
public static void CheckMetaInfo(ExtendedBlock b, DataNode dn)
{
Block metainfo = DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(b.GetBlockPoolId
(), b.GetBlockId());
NUnit.Framework.Assert.AreEqual(b.GetBlockId(), metainfo.GetBlockId());
NUnit.Framework.Assert.AreEqual(b.GetNumBytes(), metainfo.GetNumBytes());
}
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/>
private BlockReader GetBlockReaderLocal()
{
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": trying to construct a BlockReaderLocal " + "for short-circuit reads."
);
}
if (pathInfo == null)
{
pathInfo = clientContext.GetDomainSocketFactory().GetPathInfo(inetSocketAddress,
conf);
}
if (!pathInfo.GetPathState().GetUsableForShortCircuit())
{
PerformanceAdvisory.Log.Debug(this + ": " + pathInfo + " is not " + "usable for short circuit; giving up on BlockReaderLocal."
);
return(null);
}
ShortCircuitCache cache = clientContext.GetShortCircuitCache();
ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId
());
ShortCircuitReplicaInfo info = cache.FetchOrCreate(key, this);
SecretManager.InvalidToken exc = info.GetInvalidTokenException();
if (exc != null)
{
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": got InvalidToken exception while trying to " + "construct BlockReaderLocal via "
+ pathInfo.GetPath());
}
throw exc;
}
if (info.GetReplica() == null)
{
if (Log.IsTraceEnabled())
{
PerformanceAdvisory.Log.Debug(this + ": failed to get " + "ShortCircuitReplica. Cannot construct "
+ "BlockReaderLocal via " + pathInfo.GetPath());
}
return(null);
}
return(new BlockReaderLocal.Builder(conf).SetFilename(fileName).SetBlock(block).SetStartOffset
(startOffset).SetShortCircuitReplica(info.GetReplica()).SetVerifyChecksum(verifyChecksum
).SetCachingStrategy(cachingStrategy).SetStorageType(storageType).Build());
}
/// <summary>Generate a block token for a specified user</summary>
/// <exception cref="System.IO.IOException"/>
public virtual Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> GenerateToken
(string userId, ExtendedBlock block, EnumSet <BlockTokenSecretManager.AccessMode>
modes)
{
BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block.GetBlockPoolId()
, block.GetBlockId(), modes);
return(new Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier>(id, this));
}
/// <exception cref="System.IO.IOException"/>
internal static void CheckSuccess(DataTransferProtos.BlockOpResponseProto status,
Peer peer, ExtendedBlock block, string file)
{
string logInfo = "for OP_READ_BLOCK" + ", self=" + peer.GetLocalAddressString() +
", remote=" + peer.GetRemoteAddressString() + ", for file " + file + ", for pool "
+ block.GetBlockPoolId() + " block " + block.GetBlockId() + "_" + block.GetGenerationStamp
();
DataTransferProtoUtil.CheckBlockOpStatus(status, logInfo);
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Datanode.ReplicaNotFoundException"
/// />
private static Replica GetReplica(ExtendedBlock block, DataNode datanode)
{
Replica replica = datanode.data.GetReplica(block.GetBlockPoolId(), block.GetBlockId
());
if (replica == null)
{
throw new ReplicaNotFoundException(block);
}
return(replica);
}
/// <summary>Convert an ExtendedBlock to a Json map.</summary>
private static IDictionary <string, object> ToJsonMap(ExtendedBlock extendedblock)
{
if (extendedblock == null)
{
return(null);
}
IDictionary <string, object> m = new SortedDictionary <string, object>();
m["blockPoolId"] = extendedblock.GetBlockPoolId();
m["blockId"] = extendedblock.GetBlockId();
m["numBytes"] = extendedblock.GetNumBytes();
m["generationStamp"] = extendedblock.GetGenerationStamp();
return(m);
}
public virtual void TestBlockTokenRpc()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(block3.ToString
());
ticket.AddToken(token);
ClientDatanodeProtocol proxy = null;
try
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(addr, ticket, conf, NetUtils.GetDefaultSocketFactory
(conf));
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
}
finally
{
server.Stop();
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
}
/// <summary>
/// sendBlock() is used to read block and its metadata and stream the data to
/// either a client or to another datanode.
/// </summary>
/// <param name="out">stream to which the block is written to</param>
/// <param name="baseStream">
/// optional. if non-null, <code>out</code> is assumed to
/// be a wrapper over this stream. This enables optimizations for
/// sending the data, e.g.
/// <see cref="Org.Apache.Hadoop.Net.SocketOutputStream.TransferToFully(Sharpen.FileChannel, long, int)
/// "/>
/// .
/// </param>
/// <param name="throttler">for sending data.</param>
/// <returns>total bytes read, including checksum data.</returns>
/// <exception cref="System.IO.IOException"/>
internal virtual long SendBlock(DataOutputStream @out, OutputStream baseStream, DataTransferThrottler
throttler)
{
TraceScope scope = Trace.StartSpan("sendBlock_" + block.GetBlockId(), Sampler.Never
);
try
{
return(DoSendBlock(@out, baseStream, throttler));
}
finally
{
scope.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private BlockReaderLocalLegacy(DFSClient.Conf conf, string hdfsfile, ExtendedBlock
block, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, long
startOffset, long length, BlockLocalPathInfo pathinfo, DataChecksum checksum, bool
verifyChecksum, FileInputStream dataIn, long firstChunkOffset, FileInputStream
checksumIn)
{
this.filename = hdfsfile;
this.checksum = checksum;
this.verifyChecksum = verifyChecksum;
this.startOffset = Math.Max(startOffset, 0);
this.blockId = block.GetBlockId();
bytesPerChecksum = this.checksum.GetBytesPerChecksum();
checksumSize = this.checksum.GetChecksumSize();
this.dataIn = dataIn;
this.checksumIn = checksumIn;
this.offsetFromChunkBoundary = (int)(startOffset - firstChunkOffset);
int chunksPerChecksumRead = GetSlowReadBufferNumChunks(conf.shortCircuitBufferSize
, bytesPerChecksum);
slowReadBuff = bufferPool.GetBuffer(bytesPerChecksum * chunksPerChecksumRead);
checksumBuff = bufferPool.GetBuffer(checksumSize * chunksPerChecksumRead);
// Initially the buffers have nothing to read.
slowReadBuff.Flip();
checksumBuff.Flip();
bool success = false;
try
{
// Skip both input streams to beginning of the chunk containing startOffset
IOUtils.SkipFully(dataIn, firstChunkOffset);
if (checksumIn != null)
{
long checkSumOffset = (firstChunkOffset / bytesPerChecksum) * checksumSize;
IOUtils.SkipFully(checksumIn, checkSumOffset);
}
success = true;
}
finally
{
if (!success)
{
bufferPool.ReturnBuffer(slowReadBuff);
bufferPool.ReturnBuffer(checksumBuff);
}
}
}
/* fill up a cluster with <code>numNodes</code> datanodes
* whose used space to be <code>size</code>
*/
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.TimeoutException"/>
private static ExtendedBlock[][] GenerateBlocks(TestBalancerWithMultipleNameNodes.Suite
s, long size)
{
ExtendedBlock[][] blocks = new ExtendedBlock[s.clients.Length][];
for (int n = 0; n < s.clients.Length; n++)
{
long fileLen = size / s.replication;
CreateFile(s, n, fileLen);
IList <LocatedBlock> locatedBlocks = s.clients[n].GetBlockLocations(FileName, 0, fileLen
).GetLocatedBlocks();
int numOfBlocks = locatedBlocks.Count;
blocks[n] = new ExtendedBlock[numOfBlocks];
for (int i = 0; i < numOfBlocks; i++)
{
ExtendedBlock b = locatedBlocks[i].GetBlock();
blocks[n][i] = new ExtendedBlock(b.GetBlockPoolId(), b.GetBlockId(), b.GetNumBytes
(), b.GetGenerationStamp());
}
}
return(blocks);
}
/// <summary>Create a new BlockReader specifically to satisfy a read.</summary>
/// <remarks>
/// Create a new BlockReader specifically to satisfy a read.
/// This method also sends the OP_READ_BLOCK request.
/// </remarks>
/// <param name="file">File location</param>
/// <param name="block">The block object</param>
/// <param name="blockToken">The block token for security</param>
/// <param name="startOffset">The read offset, relative to block head</param>
/// <param name="len">The number of bytes to read</param>
/// <param name="bufferSize">The IO buffer size (not the client buffer size)</param>
/// <param name="verifyChecksum">Whether to verify checksum</param>
/// <param name="clientName">Client name</param>
/// <returns>New BlockReader instance, or null on error.</returns>
/// <exception cref="System.IO.IOException"/>
public static Org.Apache.Hadoop.Hdfs.RemoteBlockReader NewBlockReader(string file
, ExtendedBlock block, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier
> blockToken, long startOffset, long len, int bufferSize, bool verifyChecksum, string
clientName, Peer peer, DatanodeID datanodeID, PeerCache peerCache, CachingStrategy
cachingStrategy)
{
// in and out will be closed when sock is closed (by the caller)
DataOutputStream @out = new DataOutputStream(new BufferedOutputStream(peer.GetOutputStream
()));
new Sender(@out).ReadBlock(block, blockToken, clientName, startOffset, len, verifyChecksum
, cachingStrategy);
//
// Get bytes in block, set streams
//
DataInputStream @in = new DataInputStream(new BufferedInputStream(peer.GetInputStream
(), bufferSize));
DataTransferProtos.BlockOpResponseProto status = DataTransferProtos.BlockOpResponseProto
.ParseFrom(PBHelper.VintPrefixed(@in));
RemoteBlockReader2.CheckSuccess(status, peer, block, file);
DataTransferProtos.ReadOpChecksumInfoProto checksumInfo = status.GetReadOpChecksumInfo
();
DataChecksum checksum = DataTransferProtoUtil.FromProto(checksumInfo.GetChecksum(
));
//Warning when we get CHECKSUM_NULL?
// Read the first chunk offset.
long firstChunkOffset = checksumInfo.GetChunkOffset();
if (firstChunkOffset < 0 || firstChunkOffset > startOffset || firstChunkOffset <=
(startOffset - checksum.GetBytesPerChecksum()))
{
throw new IOException("BlockReader: error in first chunk offset (" + firstChunkOffset
+ ") startOffset is " + startOffset + " for file " + file);
}
return(new Org.Apache.Hadoop.Hdfs.RemoteBlockReader(file, block.GetBlockPoolId(),
block.GetBlockId(), @in, checksum, verifyChecksum, startOffset, firstChunkOffset
, len, peer, datanodeID, peerCache));
}
/// <summary>go to each block on the 2nd DataNode until it fails...</summary>
/// <param name="path"/>
/// <param name="size"/>
/// <exception cref="System.IO.IOException"/>
private void TriggerFailure(string path, long size)
{
NamenodeProtocols nn = cluster.GetNameNodeRpc();
IList <LocatedBlock> locatedBlocks = nn.GetBlockLocations(path, 0, size).GetLocatedBlocks
();
foreach (LocatedBlock lb in locatedBlocks)
{
DatanodeInfo dinfo = lb.GetLocations()[1];
ExtendedBlock b = lb.GetBlock();
try
{
AccessBlock(dinfo, lb);
}
catch (IOException)
{
System.Console.Out.WriteLine("Failure triggered, on block: " + b.GetBlockId() + "; corresponding volume should be removed by now"
);
break;
}
}
}
public virtual void Handle(ExtendedBlock block, IOException e)
{
FsVolumeSpi volume = scanner.volume;
if (e == null)
{
Log.Trace("Successfully scanned {} on {}", block, volume.GetBasePath());
return;
}
// If the block does not exist anymore, then it's not an error.
if (!volume.GetDataset().Contains(block))
{
Log.Debug("Volume {}: block {} is no longer in the dataset.", volume.GetBasePath(
), block);
return;
}
// If the block exists, the exception may due to a race with write:
// The BlockSender got an old block path in rbw. BlockReceiver removed
// the rbw block from rbw to finalized but BlockSender tried to open the
// file before BlockReceiver updated the VolumeMap. The state of the
// block can be changed again now, so ignore this error here. If there
// is a block really deleted by mistake, DirectoryScan should catch it.
if (e is FileNotFoundException)
{
Log.Info("Volume {}: verification failed for {} because of " + "FileNotFoundException. This may be due to a race with write."
, volume.GetBasePath(), block);
return;
}
Log.Warn("Reporting bad {} on {}", block, volume.GetBasePath());
try
{
scanner.datanode.ReportBadBlocks(block);
}
catch (IOException)
{
// This is bad, but not bad enough to shut down the scanner.
Log.Warn("Cannot report bad " + block.GetBlockId(), e);
}
}
/// <summary>Read from the block file into a buffer.</summary>
/// <remarks>
/// Read from the block file into a buffer.
/// This function overwrites checksumBuf. It will increment dataPos.
/// </remarks>
/// <param name="buf">
/// The buffer to read into. May be dataBuf.
/// The position and limit of this buffer should be set to
/// multiples of the checksum size.
/// </param>
/// <param name="canSkipChecksum">True if we can skip checksumming.</param>
/// <returns>Total bytes read. 0 on EOF.</returns>
/// <exception cref="System.IO.IOException"/>
private int FillBuffer(ByteBuffer buf, bool canSkipChecksum)
{
lock (this)
{
TraceScope scope = Trace.StartSpan("BlockReaderLocal#fillBuffer(" + block.GetBlockId
() + ")", Sampler.Never);
try
{
int total = 0;
long startDataPos = dataPos;
int startBufPos = buf.Position();
while (buf.HasRemaining())
{
int nRead = dataIn.Read(buf, dataPos);
if (nRead < 0)
{
break;
}
dataPos += nRead;
total += nRead;
}
if (canSkipChecksum)
{
FreeChecksumBufIfExists();
return(total);
}
if (total > 0)
{
try
{
buf.Limit(buf.Position());
buf.Position(startBufPos);
CreateChecksumBufIfNeeded();
int checksumsNeeded = (total + bytesPerChecksum - 1) / bytesPerChecksum;
checksumBuf.Clear();
checksumBuf.Limit(checksumsNeeded * checksumSize);
long checksumPos = BlockMetadataHeader.GetHeaderSize() + ((startDataPos / bytesPerChecksum
) * checksumSize);
while (checksumBuf.HasRemaining())
{
int nRead = checksumIn.Read(checksumBuf, checksumPos);
if (nRead < 0)
{
throw new IOException("Got unexpected checksum file EOF at " + checksumPos + ", block file position "
+ startDataPos + " for " + "block " + block + " of file " + filename);
}
checksumPos += nRead;
}
checksumBuf.Flip();
checksum.VerifyChunkedSums(buf, checksumBuf, filename, startDataPos);
}
finally
{
buf.Position(buf.Limit());
}
}
return(total);
}
finally
{
scope.Close();
}
}
}
public virtual void TestBlockSynchronization()
{
int OrgFileSize = 3000;
Configuration conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Build();
cluster.WaitActive();
//create a file
DistributedFileSystem dfs = cluster.GetFileSystem();
string filestr = "/foo";
Path filepath = new Path(filestr);
DFSTestUtil.CreateFile(dfs, filepath, OrgFileSize, ReplicationNum, 0L);
NUnit.Framework.Assert.IsTrue(dfs.Exists(filepath));
DFSTestUtil.WaitReplication(dfs, filepath, ReplicationNum);
//get block info for the last block
LocatedBlock locatedblock = TestInterDatanodeProtocol.GetLastLocatedBlock(dfs.dfs
.GetNamenode(), filestr);
DatanodeInfo[] datanodeinfos = locatedblock.GetLocations();
NUnit.Framework.Assert.AreEqual(ReplicationNum, datanodeinfos.Length);
//connect to data nodes
DataNode[] datanodes = new DataNode[ReplicationNum];
for (int i = 0; i < ReplicationNum; i++)
{
datanodes[i] = cluster.GetDataNode(datanodeinfos[i].GetIpcPort());
NUnit.Framework.Assert.IsTrue(datanodes[i] != null);
}
//verify Block Info
ExtendedBlock lastblock = locatedblock.GetBlock();
DataNode.Log.Info("newblocks=" + lastblock);
for (int i_1 = 0; i_1 < ReplicationNum; i_1++)
{
CheckMetaInfo(lastblock, datanodes[i_1]);
}
DataNode.Log.Info("dfs.dfs.clientName=" + dfs.dfs.clientName);
cluster.GetNameNodeRpc().Append(filestr, dfs.dfs.clientName, new EnumSetWritable <
CreateFlag>(EnumSet.Of(CreateFlag.Append)));
// expire lease to trigger block recovery.
WaitLeaseRecovery(cluster);
Block[] updatedmetainfo = new Block[ReplicationNum];
long oldSize = lastblock.GetNumBytes();
lastblock = TestInterDatanodeProtocol.GetLastLocatedBlock(dfs.dfs.GetNamenode(),
filestr).GetBlock();
long currentGS = lastblock.GetGenerationStamp();
for (int i_2 = 0; i_2 < ReplicationNum; i_2++)
{
updatedmetainfo[i_2] = DataNodeTestUtils.GetFSDataset(datanodes[i_2]).GetStoredBlock
(lastblock.GetBlockPoolId(), lastblock.GetBlockId());
NUnit.Framework.Assert.AreEqual(lastblock.GetBlockId(), updatedmetainfo[i_2].GetBlockId
());
NUnit.Framework.Assert.AreEqual(oldSize, updatedmetainfo[i_2].GetNumBytes());
NUnit.Framework.Assert.AreEqual(currentGS, updatedmetainfo[i_2].GetGenerationStamp
());
}
// verify that lease recovery does not occur when namenode is in safemode
System.Console.Out.WriteLine("Testing that lease recovery cannot happen during safemode."
);
filestr = "/foo.safemode";
filepath = new Path(filestr);
dfs.Create(filepath, (short)1);
cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter,
false);
NUnit.Framework.Assert.IsTrue(dfs.dfs.Exists(filestr));
DFSTestUtil.WaitReplication(dfs, filepath, (short)1);
WaitLeaseRecovery(cluster);
// verify that we still cannot recover the lease
LeaseManager lm = NameNodeAdapter.GetLeaseManager(cluster.GetNamesystem());
NUnit.Framework.Assert.IsTrue("Found " + lm.CountLease() + " lease, expected 1",
lm.CountLease() == 1);
cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave,
false);
}
public virtual void TestUpdatePipelineAfterDelete()
{
Configuration conf = new HdfsConfiguration();
Path file = new Path("/test-file");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
DFSOutputStream @out = null;
try
{
// Create a file and make sure a block is allocated for it.
@out = (DFSOutputStream)(fs.Create(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
// Create a snapshot that includes the file.
SnapshotTestHelper.CreateSnapshot((DistributedFileSystem)fs, new Path("/"), "s1");
// Grab the block info of this file for later use.
FSDataInputStream @in = null;
ExtendedBlock oldBlock = null;
try
{
@in = fs.Open(file);
oldBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
}
finally
{
IOUtils.CloseStream(@in);
}
// Allocate a new block ID/gen stamp so we can simulate pipeline
// recovery.
string clientName = ((DistributedFileSystem)fs).GetClient().GetClientName();
LocatedBlock newLocatedBlock = namenode.UpdateBlockForPipeline(oldBlock, clientName
);
ExtendedBlock newBlock = new ExtendedBlock(oldBlock.GetBlockPoolId(), oldBlock.GetBlockId
(), oldBlock.GetNumBytes(), newLocatedBlock.GetBlock().GetGenerationStamp());
// Delete the file from the present FS. It will still exist the
// previously-created snapshot. This will log an OP_DELETE for the
// file in question.
fs.Delete(file, true);
// Simulate a pipeline recovery, wherein a new block is allocated
// for the existing block, resulting in an OP_UPDATE_BLOCKS being
// logged for the file in question.
try
{
namenode.UpdatePipeline(clientName, oldBlock, newBlock, newLocatedBlock.GetLocations
(), newLocatedBlock.GetStorageIDs());
}
catch (IOException ioe)
{
// normal
GenericTestUtils.AssertExceptionContains("does not exist or it is not under construction"
, ioe);
}
// Make sure the NN can restart with the edit logs as we have them now.
cluster.RestartNameNode(true);
}
finally
{
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
/// <summary>TC11: Racing rename</summary>
/// <exception cref="System.Exception"/>
private void TestTC11(bool appendToNewBlock)
{
Path p = new Path("/TC11/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file and write one block of data. Close file.
int len1 = (int)BlockSize;
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
//b. Reopen file in "append" mode. Append half block of data.
FSDataOutputStream out_1 = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.
Append, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
int len2 = (int)BlockSize / 2;
AppendTestUtil.Write(out_1, len1, len2);
out_1.Hflush();
//c. Rename file to file.new.
Path pnew = new Path(p + ".new");
NUnit.Framework.Assert.IsTrue(fs.Rename(p, pnew));
//d. Close file handle that was opened in (b).
out_1.Close();
//check block sizes
long len = fs.GetFileStatus(pnew).GetLen();
LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(pnew.ToString
(), 0L, len);
int numblock = locatedblocks.LocatedBlockCount();
for (int i = 0; i < numblock; i++)
{
LocatedBlock lb = locatedblocks.Get(i);
ExtendedBlock blk = lb.GetBlock();
long size = lb.GetBlockSize();
if (i < numblock - 1)
{
NUnit.Framework.Assert.AreEqual(BlockSize, size);
}
foreach (DatanodeInfo datanodeinfo in lb.GetLocations())
{
DataNode dn = cluster.GetDataNode(datanodeinfo.GetIpcPort());
Block metainfo = DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(blk.GetBlockPoolId
(), blk.GetBlockId());
NUnit.Framework.Assert.AreEqual(size, metainfo.GetNumBytes());
}
}
}
public virtual void TestDataTransferProtocol()
{
Random random = new Random();
int oneMil = 1024 * 1024;
Path file = new Path("dataprotocol.dat");
int numDataNodes = 1;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, numDataNodes);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
datanode = cluster.GetFileSystem().GetDataNodeStats(HdfsConstants.DatanodeReportType
.Live)[0];
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
int fileLen = Math.Min(conf.GetInt(DFSConfigKeys.DfsBlockSizeKey, 4096), 4096);
CreateFile(fileSys, file, fileLen);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
string poolId = firstBlock.GetBlockPoolId();
long newBlockId = firstBlock.GetBlockId() + 1;
recvBuf.Reset();
sendBuf.Reset();
// bad version
recvOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
sendOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
SendRecvData("Wrong Version", true);
// bad ops
sendBuf.Reset();
sendOut.WriteShort((short)DataTransferProtocol.DataTransferVersion);
sendOut.WriteByte(OP.WriteBlock.code - 1);
SendRecvData("Wrong Op Code", true);
/* Test OP_WRITE_BLOCK */
sendBuf.Reset();
DataChecksum badChecksum = Org.Mockito.Mockito.Spy(DefaultChecksum);
Org.Mockito.Mockito.DoReturn(-1).When(badChecksum).GetBytesPerChecksum();
WriteBlock(poolId, newBlockId, badChecksum);
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, null, recvOut);
SendRecvData("wrong bytesPerChecksum while writing", true);
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
PacketHeader hdr = new PacketHeader(4, 0, 100, false, -1 - random.Next(oneMil), false
);
// size of packet
// offset in block,
// seqno
// last packet
// bad datalen
hdr.Write(sendOut);
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Error) }).Write(recvOut);
SendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, true);
// test for writing a valid zero size block
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
hdr = new PacketHeader(8, 0, 100, true, 0, false);
// size of packet
// OffsetInBlock
// sequencenumber
// lastPacketInBlock
// chunk length
hdr.Write(sendOut);
sendOut.WriteInt(0);
// zero checksum
sendOut.Flush();
//ok finally write a block with 0 len
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Success) }).Write(recvOut);
SendRecvData("Writing a zero len block blockid " + newBlockId, false);
/* Test OP_READ_BLOCK */
string bpid = cluster.GetNamesystem().GetBlockPoolId();
ExtendedBlock blk = new ExtendedBlock(bpid, firstBlock.GetLocalBlock());
long blkid = blk.GetBlockId();
// bad block id
sendBuf.Reset();
recvBuf.Reset();
blk.SetBlockId(blkid - 1);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong block ID " + newBlockId + " for read", false);
// negative block start offset -1L
sendBuf.Reset();
blk.SetBlockId(blkid);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", -1L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative start-offset for read for block " + firstBlock.GetBlockId(
), false);
// bad block start offset
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", fileLen, fileLen,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong start-offset for reading block " + firstBlock.GetBlockId(), false
);
// negative length is ok. Datanode assumes we want to read the whole block.
recvBuf.Reset();
((DataTransferProtos.BlockOpResponseProto)DataTransferProtos.BlockOpResponseProto
.NewBuilder().SetStatus(DataTransferProtos.Status.Success).SetReadOpChecksumInfo
(DataTransferProtos.ReadOpChecksumInfoProto.NewBuilder().SetChecksum(DataTransferProtoUtil
.ToProto(DefaultChecksum)).SetChunkOffset(0L)).Build()).WriteDelimitedTo(recvOut
);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, -1L - random.
Next(oneMil), true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative length for reading block " + firstBlock.GetBlockId(), false
);
// length is more than size of block.
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, "opReadBlock " + firstBlock +
" received exception java.io.IOException: " + "Offset 0 and length 4097 don't match block "
+ firstBlock + " ( blockLen 4096 )", recvOut);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen + 1,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong length for reading block " + firstBlock.GetBlockId(), false);
//At the end of all this, read the file to make sure that succeeds finally.
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
ReadFile(fileSys, file, fileLen);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>The following test first creates a file.</summary>
/// <remarks>
/// The following test first creates a file.
/// It verifies the block information from a datanode.
/// Then, it updates the block with new information and verifies again.
/// </remarks>
/// <param name="useDnHostname">whether DNs should connect to other DNs by hostname</param>
/// <exception cref="System.Exception"/>
private void CheckBlockMetaDataInfo(bool useDnHostname)
{
MiniDFSCluster cluster = null;
conf.SetBoolean(DFSConfigKeys.DfsDatanodeUseDnHostname, useDnHostname);
if (useDnHostname)
{
// Since the mini cluster only listens on the loopback we have to
// ensure the hostname used to access DNs maps to the loopback. We
// do this by telling the DN to advertise localhost as its hostname
// instead of the default hostname.
conf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, "localhost");
}
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).CheckDataNodeHostConfig
(true).Build();
cluster.WaitActive();
//create a file
DistributedFileSystem dfs = cluster.GetFileSystem();
string filestr = "/foo";
Path filepath = new Path(filestr);
DFSTestUtil.CreateFile(dfs, filepath, 1024L, (short)3, 0L);
NUnit.Framework.Assert.IsTrue(dfs.Exists(filepath));
//get block info
LocatedBlock locatedblock = GetLastLocatedBlock(DFSClientAdapter.GetDFSClient(dfs
).GetNamenode(), filestr);
DatanodeInfo[] datanodeinfo = locatedblock.GetLocations();
NUnit.Framework.Assert.IsTrue(datanodeinfo.Length > 0);
//connect to a data node
DataNode datanode = cluster.GetDataNode(datanodeinfo[0].GetIpcPort());
InterDatanodeProtocol idp = DataNodeTestUtils.CreateInterDatanodeProtocolProxy(datanode
, datanodeinfo[0], conf, useDnHostname);
// Stop the block scanners.
datanode.GetBlockScanner().RemoveAllVolumeScanners();
//verify BlockMetaDataInfo
ExtendedBlock b = locatedblock.GetBlock();
InterDatanodeProtocol.Log.Info("b=" + b + ", " + b.GetType());
CheckMetaInfo(b, datanode);
long recoveryId = b.GetGenerationStamp() + 1;
idp.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock(b, locatedblock.
GetLocations(), recoveryId));
//verify updateBlock
ExtendedBlock newblock = new ExtendedBlock(b.GetBlockPoolId(), b.GetBlockId(), b.
GetNumBytes() / 2, b.GetGenerationStamp() + 1);
idp.UpdateReplicaUnderRecovery(b, recoveryId, b.GetBlockId(), newblock.GetNumBytes
());
CheckMetaInfo(newblock, datanode);
// Verify correct null response trying to init recovery for a missing block
ExtendedBlock badBlock = new ExtendedBlock("fake-pool", b.GetBlockId(), 0, 0);
NUnit.Framework.Assert.IsNull(idp.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock
(badBlock, locatedblock.GetLocations(), recoveryId)));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Scan a block.</summary>
/// <param name="cblock">The block to scan.</param>
/// <param name="bytesPerSec">The bytes per second to scan at.</param>
/// <returns>
/// The length of the block that was scanned, or
/// -1 if the block could not be scanned.
/// </returns>
private long ScanBlock(ExtendedBlock cblock, long bytesPerSec)
{
// 'cblock' has a valid blockId and block pool id, but we don't yet know the
// genstamp the block is supposed to have. Ask the FsDatasetImpl for this
// information.
ExtendedBlock block = null;
try
{
Block b = volume.GetDataset().GetStoredBlock(cblock.GetBlockPoolId(), cblock.GetBlockId
());
if (b == null)
{
Log.Info("FileNotFound while finding block {} on volume {}", cblock, volume.GetBasePath
());
}
else
{
block = new ExtendedBlock(cblock.GetBlockPoolId(), b);
}
}
catch (FileNotFoundException)
{
Log.Info("FileNotFoundException while finding block {} on volume {}", cblock, volume
.GetBasePath());
}
catch (IOException)
{
Log.Warn("I/O error while finding block {} on volume {}", cblock, volume.GetBasePath
());
}
if (block == null)
{
return(-1);
}
// block not found.
BlockSender blockSender = null;
try
{
blockSender = new BlockSender(block, 0, -1, false, true, true, datanode, null, CachingStrategy
.NewDropBehind());
throttler.SetBandwidth(bytesPerSec);
long bytesRead = blockSender.SendBlock(nullStream, null, throttler);
resultHandler.Handle(block, null);
return(bytesRead);
}
catch (IOException e)
{
resultHandler.Handle(block, e);
}
finally
{
IOUtils.Cleanup(null, blockSender);
}
return(-1);
}
/// <exception cref="System.IO.IOException"/>
private int GetTrueReplication(MiniDFSCluster cluster, ExtendedBlock block)
{
int count = 0;
foreach (DataNode dn in cluster.GetDataNodes())
{
if (DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(block.GetBlockPoolId(), block
.GetBlockId()) != null)
{
count++;
}
}
return(count);
}
/// <summary>try to access a block on a data node.</summary>
/// <remarks>try to access a block on a data node. If fails - throws exception</remarks>
/// <param name="datanode"/>
/// <param name="lblock"/>
/// <exception cref="System.IO.IOException"/>
private void AccessBlock(DatanodeInfo datanode, LocatedBlock lblock)
{
IPEndPoint targetAddr = null;
ExtendedBlock block = lblock.GetBlock();
targetAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
BlockReader blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).SetInetSocketAddress
(targetAddr).SetBlock(block).SetFileName(BlockReaderFactory.GetFileName(targetAddr
, "test-blockpoolid", block.GetBlockId())).SetBlockToken(lblock.GetBlockToken())
.SetStartOffset(0).SetLength(-1).SetVerifyChecksum(true).SetClientName("TestDataNodeVolumeFailure"
).SetDatanodeInfo(datanode).SetCachingStrategy(CachingStrategy.NewDefaultStrategy
()).SetClientCacheContext(ClientContext.GetFromConf(conf)).SetConfiguration(conf
).SetRemotePeerFactory(new _RemotePeerFactory_422(this)).Build();
blockReader.Close();
}
/// <summary>Get a BlockReader for the given block.</summary>
/// <exception cref="System.IO.IOException"/>
public static BlockReader GetBlockReader(MiniDFSCluster cluster, LocatedBlock testBlock
, int offset, int lenToRead)
{
IPEndPoint targetAddr = null;
ExtendedBlock block = testBlock.GetBlock();
DatanodeInfo[] nodes = testBlock.GetLocations();
targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr());
DistributedFileSystem fs = cluster.GetFileSystem();
return(new BlockReaderFactory(fs.GetClient().GetConf()).SetInetSocketAddress(targetAddr
).SetBlock(block).SetFileName(targetAddr.ToString() + ":" + block.GetBlockId()).
SetBlockToken(testBlock.GetBlockToken()).SetStartOffset(offset).SetLength(lenToRead
).SetVerifyChecksum(true).SetClientName("BlockReaderTestUtil").SetDatanodeInfo(nodes
[0]).SetClientCacheContext(ClientContext.GetFromConf(fs.GetConf())).SetCachingStrategy
(CachingStrategy.NewDefaultStrategy()).SetConfiguration(fs.GetConf()).SetAllowShortCircuitLocalReads
(true).SetRemotePeerFactory(new _RemotePeerFactory_196(fs)).Build());
}
public virtual void TestUpdateReplicaUnderRecovery()
{
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
cluster.WaitActive();
string bpid = cluster.GetNamesystem().GetBlockPoolId();
//create a file
DistributedFileSystem dfs = cluster.GetFileSystem();
string filestr = "/foo";
Path filepath = new Path(filestr);
DFSTestUtil.CreateFile(dfs, filepath, 1024L, (short)3, 0L);
//get block info
LocatedBlock locatedblock = GetLastLocatedBlock(DFSClientAdapter.GetDFSClient(dfs
).GetNamenode(), filestr);
DatanodeInfo[] datanodeinfo = locatedblock.GetLocations();
NUnit.Framework.Assert.IsTrue(datanodeinfo.Length > 0);
//get DataNode and FSDataset objects
DataNode datanode = cluster.GetDataNode(datanodeinfo[0].GetIpcPort());
NUnit.Framework.Assert.IsTrue(datanode != null);
//initReplicaRecovery
ExtendedBlock b = locatedblock.GetBlock();
long recoveryid = b.GetGenerationStamp() + 1;
long newlength = b.GetNumBytes() - 1;
FsDatasetSpi <object> fsdataset = DataNodeTestUtils.GetFSDataset(datanode);
ReplicaRecoveryInfo rri = fsdataset.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock
(b, null, recoveryid));
//check replica
ReplicaInfo replica = FsDatasetTestUtil.FetchReplicaInfo(fsdataset, bpid, b.GetBlockId
());
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.ReplicaState.Rur, replica.GetState
());
//check meta data before update
FsDatasetImpl.CheckReplicaFiles(replica);
{
//case "THIS IS NOT SUPPOSED TO HAPPEN"
//with (block length) != (stored replica's on disk length).
//create a block with same id and gs but different length.
ExtendedBlock tmp = new ExtendedBlock(b.GetBlockPoolId(), rri.GetBlockId(), rri.GetNumBytes
() - 1, rri.GetGenerationStamp());
try
{
//update should fail
fsdataset.UpdateReplicaUnderRecovery(tmp, recoveryid, tmp.GetBlockId(), newlength
);
NUnit.Framework.Assert.Fail();
}
catch (IOException ioe)
{
System.Console.Out.WriteLine("GOOD: getting " + ioe);
}
}
//update
string storageID = fsdataset.UpdateReplicaUnderRecovery(new ExtendedBlock(b.GetBlockPoolId
(), rri), recoveryid, rri.GetBlockId(), newlength);
NUnit.Framework.Assert.IsTrue(storageID != null);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest
test, bool checksum, long readahead)
{
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
MiniDFSCluster cluster = null;
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum
);
conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest
.BytesPerChecksum);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead);
test.SetConfiguration(conf);
FileInputStream dataIn = null;
FileInputStream metaIn = null;
Path TestPath = new Path("/a");
long RandomSeed = 4567L;
BlockReaderLocal blockReaderLocal = null;
FSDataInputStream fsIn = null;
byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength];
FileSystem fs = null;
ShortCircuitShm shm = null;
RandomAccessFile raf = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
);
fsIn.Close();
fsIn = null;
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath);
FilePath dataFile = cluster.GetBlockFile(0, block);
FilePath metaFile = cluster.GetBlockMetadataFile(0, block);
ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache
();
cluster.Shutdown();
cluster = null;
test.Setup(dataFile, checksum);
FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile)
, new FileInputStream(metaFile) };
dataIn = streams[0];
metaIn = streams[1];
ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId
());
raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID.
RandomUUID().ToString()), "rw");
raf.SetLength(8192);
FileInputStream shmStream = new FileInputStream(raf.GetFD());
shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream);
ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache
, Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block))
);
blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename
(TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy
(new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build();
dataIn = null;
metaIn = null;
test.DoTest(blockReaderLocal, original);
// BlockReaderLocal should not alter the file position.
NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position());
NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position());
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
if (dataIn != null)
{
dataIn.Close();
}
if (metaIn != null)
{
metaIn.Close();
}
if (blockReaderLocal != null)
{
blockReaderLocal.Close();
}
if (shm != null)
{
shm.Free();
}
if (raf != null)
{
raf.Close();
}
}
}
public virtual void TestOpWrite()
{
int numDataNodes = 1;
long BlockIdFudge = 128;
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
string poolId = cluster.GetNamesystem().GetBlockPoolId();
datanode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes()[0], poolId
);
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
/* Test writing to finalized replicas */
Path file = new Path("dataprotocol.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
// test PIPELINE_SETUP_CREATE on a finalized block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create an existing block"
, true);
// test PIPELINE_DATA_STREAMING on a finalized block
TestWrite(firstBlock, BlockConstructionStage.DataStreaming, 0L, "Unexpected stage"
, true);
// test PIPELINE_SETUP_STREAMING_RECOVERY on an existing block
long newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Cannot recover data streaming to a finalized replica", true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Append to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_APPEND_RECOVERY on an existing block
file = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover appending to a finalized replica", false);
// test PIPELINE_CLOSE_RECOVERY on an existing block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineCloseRecovery, newGS, "Recover failed close to a finalized replica"
, false);
firstBlock.SetGenerationStamp(newGS);
// Test writing to a new block. Don't choose the next sequential
// block ID to avoid conflicting with IDs chosen by the NN.
long newBlockId = firstBlock.GetBlockId() + BlockIdFudge;
ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId
, 0, firstBlock.GetGenerationStamp());
// test PIPELINE_SETUP_CREATE on a new block
TestWrite(newBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Create a new block"
, false);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a new block
newGS = newBlock.GetGenerationStamp() + 1;
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
TestWrite(newBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS,
"Recover a new block", true);
// test PIPELINE_SETUP_APPEND on a new block
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a new block"
, true);
// test PIPELINE_SETUP_APPEND_RECOVERY on a new block
newBlock.SetBlockId(newBlock.GetBlockId() + 1);
newGS = newBlock.GetGenerationStamp() + 1;
TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Cannot append to a new block"
, true);
/* Test writing to RBW replicas */
Path file1 = new Path("dataprotocol1.dat");
DFSTestUtil.CreateFile(fileSys, file1, 1L, (short)numDataNodes, 0L);
DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file1).GetWrappedStream()
);
@out.Write(1);
@out.Hflush();
FSDataInputStream @in = fileSys.Open(file1);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
try
{
// test PIPELINE_SETUP_CREATE on a RBW block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create a RBW block"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a RBW replica"
, true);
// test PIPELINE_SETUP_APPEND on an existing block
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS,
"Recover append to a RBW replica", false);
firstBlock.SetGenerationStamp(newGS);
// test PIPELINE_SETUP_STREAMING_RECOVERY on a RBW block
file = new Path("dataprotocol2.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
@in = fileSys.Open(file);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
firstBlock.SetNumBytes(2L);
newGS = firstBlock.GetGenerationStamp() + 1;
TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS
, "Recover a RBW replica", false);
}
finally
{
IOUtils.CloseStream(@in);
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestGetNewStamp()
{
int numDataNodes = 1;
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
FileSystem fileSys = cluster.GetFileSystem();
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
/* Test writing to finalized replicas */
Path file = new Path("dataprotocol.dat");
DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
// test getNewStampAndToken on a finalized block
try
{
namenode.UpdateBlockForPipeline(firstBlock, string.Empty);
NUnit.Framework.Assert.Fail("Can not get a new GS from a finalized block");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("is not under Construction"));
}
// test getNewStampAndToken on a non-existent block
try
{
long newBlockId = firstBlock.GetBlockId() + 1;
ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId
, 0, firstBlock.GetGenerationStamp());
namenode.UpdateBlockForPipeline(newBlock, string.Empty);
NUnit.Framework.Assert.Fail("Cannot get a new GS from a non-existent block");
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Contains("does not exist"));
}
/* Test RBW replicas */
// change first block to a RBW
DFSOutputStream @out = null;
try
{
@out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream());
@out.Write(1);
@out.Hflush();
FSDataInputStream @in = null;
try
{
@in = fileSys.Open(file);
firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
}
finally
{
IOUtils.CloseStream(@in);
}
// test non-lease holder
DFSClient dfs = ((DistributedFileSystem)fileSys).dfs;
try
{
namenode.UpdateBlockForPipeline(firstBlock, "test" + dfs.clientName);
NUnit.Framework.Assert.Fail("Cannot get a new GS for a non lease holder");
}
catch (LeaseExpiredException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.StartsWith("Lease mismatch"));
}
// test null lease holder
try
{
namenode.UpdateBlockForPipeline(firstBlock, null);
NUnit.Framework.Assert.Fail("Cannot get a new GS for a null lease holder");
}
catch (LeaseExpiredException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.StartsWith("Lease mismatch"));
}
// test getNewStampAndToken on a rbw block
namenode.UpdateBlockForPipeline(firstBlock, dfs.clientName);
}
finally
{
IOUtils.CloseStream(@out);
}
}
finally
{
cluster.Shutdown();
}
}
public static Org.Apache.Hadoop.Hdfs.ExtendedBlockId FromExtendedBlock(ExtendedBlock
block)
{
return(new Org.Apache.Hadoop.Hdfs.ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId
()));
}
