/// <exception cref="System.IO.IOException"/>
internal virtual void WriteBlock(ExtendedBlock block, BlockConstructionStage stage
, long newGS, DataChecksum checksum)
{
sender.WriteBlock(block, StorageType.Default, BlockTokenSecretManager.DummyToken,
"cl", new DatanodeInfo[1], new StorageType[1], null, stage, 0, block.GetNumBytes
(), block.GetNumBytes(), newGS, checksum, CachingStrategy.NewDefaultStrategy(),
false, false, null);
}
// try reading a block using a BlockReader directly
private static void TryRead(Configuration conf, LocatedBlock lblock, bool shouldSucceed
)
{
IPEndPoint targetAddr = null;
IOException ioe = null;
BlockReader blockReader = null;
ExtendedBlock block = lblock.GetBlock();
try
{
DatanodeInfo[] nodes = lblock.GetLocations();
targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr());
blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).SetFileName(BlockReaderFactory
.GetFileName(targetAddr, "test-blockpoolid", block.GetBlockId())).SetBlock(block
).SetBlockToken(lblock.GetBlockToken()).SetInetSocketAddress(targetAddr).SetStartOffset
(0).SetLength(-1).SetVerifyChecksum(true).SetClientName("TestBlockTokenWithDFS")
.SetDatanodeInfo(nodes[0]).SetCachingStrategy(CachingStrategy.NewDefaultStrategy
()).SetClientCacheContext(ClientContext.GetFromConf(conf)).SetConfiguration(conf
).SetRemotePeerFactory(new _RemotePeerFactory_162(conf)).Build();
}
catch (IOException ex)
{
ioe = ex;
}
finally
{
if (blockReader != null)
{
try
{
blockReader.Close();
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}
if (shouldSucceed)
{
NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is invalid, " + "when it is expected to be valid"
, blockReader);
}
else
{
NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is valid, " + "when it is expected to be invalid"
, ioe);
NUnit.Framework.Assert.IsTrue("OP_READ_BLOCK failed due to reasons other than access token: "
, ioe is InvalidBlockTokenException);
}
}
/// <summary>Get a BlockReader for the given block.</summary>
/// <exception cref="System.IO.IOException"/>
public static BlockReader GetBlockReader(MiniDFSCluster cluster, LocatedBlock testBlock
, int offset, int lenToRead)
{
IPEndPoint targetAddr = null;
ExtendedBlock block = testBlock.GetBlock();
DatanodeInfo[] nodes = testBlock.GetLocations();
targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr());
DistributedFileSystem fs = cluster.GetFileSystem();
return(new BlockReaderFactory(fs.GetClient().GetConf()).SetInetSocketAddress(targetAddr
).SetBlock(block).SetFileName(targetAddr.ToString() + ":" + block.GetBlockId()).
SetBlockToken(testBlock.GetBlockToken()).SetStartOffset(offset).SetLength(lenToRead
).SetVerifyChecksum(true).SetClientName("BlockReaderTestUtil").SetDatanodeInfo(nodes
[0]).SetClientCacheContext(ClientContext.GetFromConf(fs.GetConf())).SetCachingStrategy
(CachingStrategy.NewDefaultStrategy()).SetConfiguration(fs.GetConf()).SetAllowShortCircuitLocalReads
(true).SetRemotePeerFactory(new _RemotePeerFactory_196(fs)).Build());
}
public virtual void TestDataTransferProtocol()
{
Random random = new Random();
int oneMil = 1024 * 1024;
Path file = new Path("dataprotocol.dat");
int numDataNodes = 1;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, numDataNodes);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
datanode = cluster.GetFileSystem().GetDataNodeStats(HdfsConstants.DatanodeReportType
.Live)[0];
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
int fileLen = Math.Min(conf.GetInt(DFSConfigKeys.DfsBlockSizeKey, 4096), 4096);
CreateFile(fileSys, file, fileLen);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
string poolId = firstBlock.GetBlockPoolId();
long newBlockId = firstBlock.GetBlockId() + 1;
recvBuf.Reset();
sendBuf.Reset();
// bad version
recvOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
sendOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
SendRecvData("Wrong Version", true);
// bad ops
sendBuf.Reset();
sendOut.WriteShort((short)DataTransferProtocol.DataTransferVersion);
sendOut.WriteByte(OP.WriteBlock.code - 1);
SendRecvData("Wrong Op Code", true);
/* Test OP_WRITE_BLOCK */
sendBuf.Reset();
DataChecksum badChecksum = Org.Mockito.Mockito.Spy(DefaultChecksum);
Org.Mockito.Mockito.DoReturn(-1).When(badChecksum).GetBytesPerChecksum();
WriteBlock(poolId, newBlockId, badChecksum);
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, null, recvOut);
SendRecvData("wrong bytesPerChecksum while writing", true);
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
PacketHeader hdr = new PacketHeader(4, 0, 100, false, -1 - random.Next(oneMil), false
);
// size of packet
// offset in block,
// seqno
// last packet
// bad datalen
hdr.Write(sendOut);
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Error) }).Write(recvOut);
SendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, true);
// test for writing a valid zero size block
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
hdr = new PacketHeader(8, 0, 100, true, 0, false);
// size of packet
// OffsetInBlock
// sequencenumber
// lastPacketInBlock
// chunk length
hdr.Write(sendOut);
sendOut.WriteInt(0);
// zero checksum
sendOut.Flush();
//ok finally write a block with 0 len
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Success) }).Write(recvOut);
SendRecvData("Writing a zero len block blockid " + newBlockId, false);
/* Test OP_READ_BLOCK */
string bpid = cluster.GetNamesystem().GetBlockPoolId();
ExtendedBlock blk = new ExtendedBlock(bpid, firstBlock.GetLocalBlock());
long blkid = blk.GetBlockId();
// bad block id
sendBuf.Reset();
recvBuf.Reset();
blk.SetBlockId(blkid - 1);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong block ID " + newBlockId + " for read", false);
// negative block start offset -1L
sendBuf.Reset();
blk.SetBlockId(blkid);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", -1L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative start-offset for read for block " + firstBlock.GetBlockId(
), false);
// bad block start offset
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", fileLen, fileLen,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong start-offset for reading block " + firstBlock.GetBlockId(), false
);
// negative length is ok. Datanode assumes we want to read the whole block.
recvBuf.Reset();
((DataTransferProtos.BlockOpResponseProto)DataTransferProtos.BlockOpResponseProto
.NewBuilder().SetStatus(DataTransferProtos.Status.Success).SetReadOpChecksumInfo
(DataTransferProtos.ReadOpChecksumInfoProto.NewBuilder().SetChecksum(DataTransferProtoUtil
.ToProto(DefaultChecksum)).SetChunkOffset(0L)).Build()).WriteDelimitedTo(recvOut
);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, -1L - random.
Next(oneMil), true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative length for reading block " + firstBlock.GetBlockId(), false
);
// length is more than size of block.
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, "opReadBlock " + firstBlock +
" received exception java.io.IOException: " + "Offset 0 and length 4097 don't match block "
+ firstBlock + " ( blockLen 4096 )", recvOut);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen + 1,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong length for reading block " + firstBlock.GetBlockId(), false);
//At the end of all this, read the file to make sure that succeeds finally.
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
ReadFile(fileSys, file, fileLen);
}
finally
{
cluster.Shutdown();
}
}
/// <summary>Receive OP_WRITE_BLOCK</summary>
/// <exception cref="System.IO.IOException"/>
private void OpWriteBlock(DataInputStream @in)
{
DataTransferProtos.OpWriteBlockProto proto = DataTransferProtos.OpWriteBlockProto
.ParseFrom(PBHelper.VintPrefixed(@in));
DatanodeInfo[] targets = PBHelper.Convert(proto.GetTargetsList());
TraceScope traceScope = DataTransferProtoUtil.ContinueTraceSpan(proto.GetHeader()
, proto.GetType().Name);
try
{
WriteBlock(PBHelper.Convert(proto.GetHeader().GetBaseHeader().GetBlock()), PBHelper
.ConvertStorageType(proto.GetStorageType()), PBHelper.Convert(proto.GetHeader().
GetBaseHeader().GetToken()), proto.GetHeader().GetClientName(), targets, PBHelper
.ConvertStorageTypes(proto.GetTargetStorageTypesList(), targets.Length), PBHelper
.Convert(proto.GetSource()), DataTransferProtoUtil.FromProto(proto.GetStage()),
proto.GetPipelineSize(), proto.GetMinBytesRcvd(), proto.GetMaxBytesRcvd(), proto
.GetLatestGenerationStamp(), DataTransferProtoUtil.FromProto(proto.GetRequestedChecksum
()), (proto.HasCachingStrategy() ? GetCachingStrategy(proto.GetCachingStrategy()
) : CachingStrategy.NewDefaultStrategy()), (proto.HasAllowLazyPersist() ? proto.
GetAllowLazyPersist() : false), (proto.HasPinning() ? proto.GetPinning() : false
), (PBHelper.ConvertBooleanList(proto.GetTargetPinningsList())));
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
}
/// <summary>Receive OP_READ_BLOCK</summary>
/// <exception cref="System.IO.IOException"/>
private void OpReadBlock()
{
DataTransferProtos.OpReadBlockProto proto = DataTransferProtos.OpReadBlockProto.ParseFrom
(PBHelper.VintPrefixed(@in));
TraceScope traceScope = DataTransferProtoUtil.ContinueTraceSpan(proto.GetHeader()
, proto.GetType().Name);
try
{
ReadBlock(PBHelper.Convert(proto.GetHeader().GetBaseHeader().GetBlock()), PBHelper
.Convert(proto.GetHeader().GetBaseHeader().GetToken()), proto.GetHeader().GetClientName
(), proto.GetOffset(), proto.GetLen(), proto.GetSendChecksums(), (proto.HasCachingStrategy
() ? GetCachingStrategy(proto.GetCachingStrategy()) : CachingStrategy.NewDefaultStrategy
()));
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
}