public static DataTransferProtos.ChecksumProto ToProto(DataChecksum checksum)
{
HdfsProtos.ChecksumTypeProto type = PBHelper.Convert(checksum.GetChecksumType());
// ChecksumType#valueOf never returns null
return((DataTransferProtos.ChecksumProto)DataTransferProtos.ChecksumProto.NewBuilder
().SetBytesPerChecksum(checksum.GetBytesPerChecksum()).SetType(type).Build());
}
private RemoteBlockReader(string file, string bpid, long blockId, DataInputStream
@in, DataChecksum checksum, bool verifyChecksum, long startOffset, long firstChunkOffset
, long bytesToRead, Peer peer, DatanodeID datanodeID, PeerCache peerCache)
: base(new Path("/" + Block.BlockFilePrefix + blockId + ":" + bpid + ":of:" + file
), 1, verifyChecksum, checksum.GetChecksumSize() > 0 ? checksum : null, checksum
.GetBytesPerChecksum(), checksum.GetChecksumSize())
{
// Path is used only for printing block and file information in debug
/*too non path-like?*/
this.isLocal = DFSClient.IsLocalAddress(NetUtils.CreateSocketAddr(datanodeID.GetXferAddr
()));
this.peer = peer;
this.datanodeID = datanodeID;
this.@in = @in;
this.checksum = checksum;
this.startOffset = Math.Max(startOffset, 0);
this.blockId = blockId;
// The total number of bytes that we need to transfer from the DN is
// the amount that the user wants (bytesToRead), plus the padding at
// the beginning in order to chunk-align. Note that the DN may elect
// to send more than this amount if the read starts/ends mid-chunk.
this.bytesNeededToFinish = bytesToRead + (startOffset - firstChunkOffset);
this.firstChunkOffset = firstChunkOffset;
lastChunkOffset = firstChunkOffset;
lastChunkLen = -1;
bytesPerChecksum = this.checksum.GetBytesPerChecksum();
checksumSize = this.checksum.GetChecksumSize();
this.peerCache = peerCache;
}
/// <exception cref="System.IO.IOException"/>
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
: base(file, fs.GetFileStatus(file).GetReplication())
{
this.datas = fs.GetRawFileSystem().Open(file, bufferSize);
this.fs = fs;
Path sumFile = fs.GetChecksumFile(file);
try
{
int sumBufferSize = fs.GetSumBufferSize(fs.GetBytesPerSum(), bufferSize);
sums = fs.GetRawFileSystem().Open(sumFile, sumBufferSize);
byte[] version = new byte[ChecksumVersion.Length];
sums.ReadFully(version);
if (!Arrays.Equals(version, ChecksumVersion))
{
throw new IOException("Not a checksum file: " + sumFile);
}
this.bytesPerSum = sums.ReadInt();
Set(fs.verifyChecksum, DataChecksum.NewCrc32(), bytesPerSum, 4);
}
catch (FileNotFoundException)
{
// quietly ignore
Set(fs.verifyChecksum, null, 1, 0);
}
catch (IOException e)
{
// loudly ignore
Log.Warn("Problem opening checksum file: " + file + ". Ignoring exception: ", e);
Set(fs.verifyChecksum, null, 1, 0);
}
}
private BlockReaderLocal(BlockReaderLocal.Builder builder)
{
this.replica = builder.replica;
this.dataIn = replica.GetDataStream().GetChannel();
this.dataPos = builder.dataPos;
this.checksumIn = replica.GetMetaStream().GetChannel();
BlockMetadataHeader header = builder.replica.GetMetaHeader();
this.checksum = header.GetChecksum();
this.verifyChecksum = builder.verifyChecksum && (this.checksum.GetChecksumType().
id != DataChecksum.ChecksumNull);
this.filename = builder.filename;
this.block = builder.block;
this.bytesPerChecksum = checksum.GetBytesPerChecksum();
this.checksumSize = checksum.GetChecksumSize();
this.maxAllocatedChunks = (bytesPerChecksum == 0) ? 0 : ((builder.bufferSize + bytesPerChecksum
- 1) / bytesPerChecksum);
// Calculate the effective maximum readahead.
// We can't do more readahead than there is space in the buffer.
int maxReadaheadChunks = (bytesPerChecksum == 0) ? 0 : ((Math.Min(builder.bufferSize
, builder.maxReadahead) + bytesPerChecksum - 1) / bytesPerChecksum);
if (maxReadaheadChunks == 0)
{
this.zeroReadaheadRequested = true;
maxReadaheadChunks = 1;
}
else
{
this.zeroReadaheadRequested = false;
}
this.maxReadaheadLength = maxReadaheadChunks * bytesPerChecksum;
this.storageType = builder.storageType;
}
public virtual void TestGetMetaData()
{
SimulatedFSDataset fsdataset = GetSimulatedFSDataset();
ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0);
try
{
NUnit.Framework.Assert.IsTrue(fsdataset.GetMetaDataInputStream(b) == null);
NUnit.Framework.Assert.IsTrue("Expected an IO exception", false);
}
catch (IOException)
{
}
// ok - as expected
AddSomeBlocks(fsdataset);
// Only need to add one but ....
b = new ExtendedBlock(bpid, 1, 0, 0);
InputStream metaInput = fsdataset.GetMetaDataInputStream(b);
DataInputStream metaDataInput = new DataInputStream(metaInput);
short version = metaDataInput.ReadShort();
NUnit.Framework.Assert.AreEqual(BlockMetadataHeader.Version, version);
DataChecksum checksum = DataChecksum.NewDataChecksum(metaDataInput);
NUnit.Framework.Assert.AreEqual(DataChecksum.Type.Null, checksum.GetChecksumType(
));
NUnit.Framework.Assert.AreEqual(0, checksum.GetChecksumSize());
}
/// <exception cref="System.IO.IOException"/>
private BlockReaderLocalLegacy(DFSClient.Conf conf, string hdfsfile, ExtendedBlock
block, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, long
startOffset, long length, BlockLocalPathInfo pathinfo, FileInputStream dataIn)
: this(conf, hdfsfile, block, token, startOffset, length, pathinfo, DataChecksum.
NewDataChecksum(DataChecksum.Type.Null, 4), false, dataIn, startOffset, null)
{
}
// Version is already read.
/// <exception cref="System.IO.IOException"/>
private static Org.Apache.Hadoop.Hdfs.Server.Datanode.BlockMetadataHeader ReadHeader
(short version, DataInputStream @in)
{
DataChecksum checksum = DataChecksum.NewDataChecksum(@in);
return(new Org.Apache.Hadoop.Hdfs.Server.Datanode.BlockMetadataHeader(version, checksum
));
}
/// <summary>
/// Create an object with a data output stream, a checksum output stream
/// and a checksum.
/// </summary>
public ReplicaOutputStreams(OutputStream dataOut, OutputStream checksumOut, DataChecksum
checksum, bool isTransientStorage)
{
this.dataOut = dataOut;
this.checksumOut = checksumOut;
this.checksum = checksum;
this.isTransientStorage = isTransientStorage;
}
/// <exception cref="System.IO.IOException"/>
internal virtual void WriteBlock(ExtendedBlock block, BlockConstructionStage stage
, long newGS, DataChecksum checksum)
{
sender.WriteBlock(block, StorageType.Default, BlockTokenSecretManager.DummyToken,
"cl", new DatanodeInfo[1], new StorageType[1], null, stage, 0, block.GetNumBytes
(), block.GetNumBytes(), newGS, checksum, CachingStrategy.NewDefaultStrategy(),
false, false, null);
}
public virtual void TestReplicationError()
{
// create a file of replication factor of 1
Path fileName = new Path("/test.txt");
int fileLen = 1;
DFSTestUtil.CreateFile(fs, fileName, 1, (short)1, 1L);
DFSTestUtil.WaitReplication(fs, fileName, (short)1);
// get the block belonged to the created file
LocatedBlocks blocks = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), fileName
.ToString(), 0, (long)fileLen);
NUnit.Framework.Assert.AreEqual("Should only find 1 block", blocks.LocatedBlockCount
(), 1);
LocatedBlock block = blocks.Get(0);
// bring up a second datanode
cluster.StartDataNodes(conf, 1, true, null, null);
cluster.WaitActive();
int sndNode = 1;
DataNode datanode = cluster.GetDataNodes()[sndNode];
// replicate the block to the second datanode
IPEndPoint target = datanode.GetXferAddress();
Socket s = Sharpen.Extensions.CreateSocket(target.Address, target.Port);
// write the header.
DataOutputStream @out = new DataOutputStream(s.GetOutputStream());
DataChecksum checksum = DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, 512
);
new Sender(@out).WriteBlock(block.GetBlock(), StorageType.Default, BlockTokenSecretManager
.DummyToken, string.Empty, new DatanodeInfo[0], new StorageType[0], null, BlockConstructionStage
.PipelineSetupCreate, 1, 0L, 0L, 0L, checksum, CachingStrategy.NewDefaultStrategy
(), false, false, null);
@out.Flush();
// close the connection before sending the content of the block
@out.Close();
// the temporary block & meta files should be deleted
string bpid = cluster.GetNamesystem().GetBlockPoolId();
FilePath storageDir = cluster.GetInstanceStorageDir(sndNode, 0);
FilePath dir1 = MiniDFSCluster.GetRbwDir(storageDir, bpid);
storageDir = cluster.GetInstanceStorageDir(sndNode, 1);
FilePath dir2 = MiniDFSCluster.GetRbwDir(storageDir, bpid);
while (dir1.ListFiles().Length != 0 || dir2.ListFiles().Length != 0)
{
Sharpen.Thread.Sleep(100);
}
// then increase the file's replication factor
fs.SetReplication(fileName, (short)2);
// replication should succeed
DFSTestUtil.WaitReplication(fs, fileName, (short)1);
// clean up the file
fs.Delete(fileName, false);
}
public static DataChecksum FromProto(DataTransferProtos.ChecksumProto proto)
{
if (proto == null)
{
return(null);
}
int bytesPerChecksum = proto.GetBytesPerChecksum();
DataChecksum.Type type = PBHelper.Convert(proto.GetType());
return(DataChecksum.NewDataChecksum(type, bytesPerChecksum));
}
/// <summary>Verifies the block's checksum.</summary>
/// <remarks>Verifies the block's checksum. This is an I/O intensive operation.</remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.FS.ChecksumException"/>
private static void VerifyChecksum(long length, FileInputStream metaIn, FileChannel
blockChannel, string blockFileName)
{
// Verify the checksum from the block's meta file
// Get the DataChecksum from the meta file header
BlockMetadataHeader header = BlockMetadataHeader.ReadHeader(new DataInputStream(new
BufferedInputStream(metaIn, BlockMetadataHeader.GetHeaderSize())));
FileChannel metaChannel = null;
try
{
metaChannel = metaIn.GetChannel();
if (metaChannel == null)
{
throw new IOException("Block InputStream meta file has no FileChannel.");
}
DataChecksum checksum = header.GetChecksum();
int bytesPerChecksum = checksum.GetBytesPerChecksum();
int checksumSize = checksum.GetChecksumSize();
int numChunks = (8 * 1024 * 1024) / bytesPerChecksum;
ByteBuffer blockBuf = ByteBuffer.Allocate(numChunks * bytesPerChecksum);
ByteBuffer checksumBuf = ByteBuffer.Allocate(numChunks * checksumSize);
// Verify the checksum
int bytesVerified = 0;
while (bytesVerified < length)
{
Preconditions.CheckState(bytesVerified % bytesPerChecksum == 0, "Unexpected partial chunk before EOF"
);
System.Diagnostics.Debug.Assert(bytesVerified % bytesPerChecksum == 0);
int bytesRead = FillBuffer(blockChannel, blockBuf);
if (bytesRead == -1)
{
throw new IOException("checksum verification failed: premature EOF");
}
blockBuf.Flip();
// Number of read chunks, including partial chunk at end
int chunks = (bytesRead + bytesPerChecksum - 1) / bytesPerChecksum;
checksumBuf.Limit(chunks * checksumSize);
FillBuffer(metaChannel, checksumBuf);
checksumBuf.Flip();
checksum.VerifyChunkedSums(blockBuf, checksumBuf, blockFileName, bytesVerified);
// Success
bytesVerified += bytesRead;
blockBuf.Clear();
checksumBuf.Clear();
}
}
finally
{
IOUtils.CloseQuietly(metaChannel);
}
}
protected internal FSOutputSummer(DataChecksum sum)
{
// data checksum
// internal buffer for storing data before it is checksumed
// internal buffer for storing checksum
// The number of valid bytes in the buffer.
// We want this value to be a multiple of 3 because the native code checksums
// 3 chunks simultaneously. The chosen value of 9 strikes a balance between
// limiting the number of JNI calls and flushing to the underlying stream
// relatively frequently.
this.sum = sum;
this.buf = new byte[sum.GetBytesPerChecksum() * BufferNumChunks];
this.checksum = new byte[GetChecksumSize() * BufferNumChunks];
this.count = 0;
}
/// <exception cref="System.IO.IOException"/>
public override void Close()
{
lock (this)
{
startOffset = -1;
checksum = null;
if (peerCache != null & sentStatusCode)
{
peerCache.Put(datanodeID, peer);
}
else
{
peer.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Close()
{
lock (this)
{
packetReceiver.Close();
startOffset = -1;
checksum = null;
if (peerCache != null && sentStatusCode)
{
peerCache.Put(datanodeID, peer);
}
else
{
peer.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, bool overwrite, int
bufferSize, short replication, long blockSize, Progressable progress, FsPermission
permission)
: base(DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, fs.GetBytesPerSum())
)
{
int bytesPerSum = fs.GetBytesPerSum();
this.datas = fs.GetRawFileSystem().Create(file, permission, overwrite, bufferSize
, replication, blockSize, progress);
int sumBufferSize = fs.GetSumBufferSize(bytesPerSum, bufferSize);
this.sums = fs.GetRawFileSystem().Create(fs.GetChecksumFile(file), permission, true
, sumBufferSize, replication, blockSize, null);
sums.Write(ChecksumVersion, 0, ChecksumVersion.Length);
sums.WriteInt(bytesPerSum);
}
/// <exception cref="System.IO.IOException"/>
private BlockReaderLocalLegacy(DFSClient.Conf conf, string hdfsfile, ExtendedBlock
block, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, long
startOffset, long length, BlockLocalPathInfo pathinfo, DataChecksum checksum, bool
verifyChecksum, FileInputStream dataIn, long firstChunkOffset, FileInputStream
checksumIn)
{
this.filename = hdfsfile;
this.checksum = checksum;
this.verifyChecksum = verifyChecksum;
this.startOffset = Math.Max(startOffset, 0);
this.blockId = block.GetBlockId();
bytesPerChecksum = this.checksum.GetBytesPerChecksum();
checksumSize = this.checksum.GetChecksumSize();
this.dataIn = dataIn;
this.checksumIn = checksumIn;
this.offsetFromChunkBoundary = (int)(startOffset - firstChunkOffset);
int chunksPerChecksumRead = GetSlowReadBufferNumChunks(conf.shortCircuitBufferSize
, bytesPerChecksum);
slowReadBuff = bufferPool.GetBuffer(bytesPerChecksum * chunksPerChecksumRead);
checksumBuff = bufferPool.GetBuffer(checksumSize * chunksPerChecksumRead);
// Initially the buffers have nothing to read.
slowReadBuff.Flip();
checksumBuff.Flip();
bool success = false;
try
{
// Skip both input streams to beginning of the chunk containing startOffset
IOUtils.SkipFully(dataIn, firstChunkOffset);
if (checksumIn != null)
{
long checkSumOffset = (firstChunkOffset / bytesPerChecksum) * checksumSize;
IOUtils.SkipFully(checksumIn, checkSumOffset);
}
success = true;
}
finally
{
if (!success)
{
bufferPool.ReturnBuffer(slowReadBuff);
bufferPool.ReturnBuffer(checksumBuff);
}
}
}
/// <summary>Create a checksum input stream that reads</summary>
/// <param name="in">The input stream to be verified for checksum.</param>
/// <param name="len">The length of the input stream including checksum bytes.</param>
public IFileInputStream(InputStream @in, long len, Configuration conf)
{
//The input stream to be verified for checksum.
// the file descriptor, if it is known
//The total length of the input file
this.@in = @in;
this.inFd = GetFileDescriptorIfAvail(@in);
sum = DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, int.MaxValue);
checksumSize = sum.GetChecksumSize();
length = len;
dataLength = length - checksumSize;
conf = (conf != null) ? conf : new Configuration();
readahead = conf.GetBoolean(MRConfig.MapredIfileReadahead, MRConfig.DefaultMapredIfileReadahead
);
readaheadLength = conf.GetInt(MRConfig.MapredIfileReadaheadBytes, MRConfig.DefaultMapredIfileReadaheadBytes
);
DoReadahead();
}
/// <exception cref="System.IO.IOException"/>
public virtual void Close()
{
lock (this)
{
IOUtils.Cleanup(Log, dataIn, checksumIn);
if (slowReadBuff != null)
{
bufferPool.ReturnBuffer(slowReadBuff);
slowReadBuff = null;
}
if (checksumBuff != null)
{
bufferPool.ReturnBuffer(checksumBuff);
checksumBuff = null;
}
startOffset = -1;
checksum = null;
}
}
/// <summary>Read the header without changing the position of the FileChannel.</summary>
/// <param name="fc">The FileChannel to read.</param>
/// <returns>the Metadata Header.</returns>
/// <exception cref="System.IO.IOException">on error.</exception>
public static Org.Apache.Hadoop.Hdfs.Server.Datanode.BlockMetadataHeader PreadHeader
(FileChannel fc)
{
byte[] arr = new byte[GetHeaderSize()];
ByteBuffer buf = ByteBuffer.Wrap(arr);
while (buf.HasRemaining())
{
if (fc.Read(buf, 0) <= 0)
{
throw new EOFException("unexpected EOF while reading " + "metadata file header");
}
}
short version = (short)((arr[0] << 8) | (arr[1] & unchecked ((int)(0xff))));
DataChecksum dataChecksum = DataChecksum.NewDataChecksum(arr, 2);
return(new Org.Apache.Hadoop.Hdfs.Server.Datanode.BlockMetadataHeader(version, dataChecksum
));
}
/// <summary>Create a new BlockReader specifically to satisfy a read.</summary>
/// <remarks>
/// Create a new BlockReader specifically to satisfy a read.
/// This method also sends the OP_READ_BLOCK request.
/// </remarks>
/// <param name="file">File location</param>
/// <param name="block">The block object</param>
/// <param name="blockToken">The block token for security</param>
/// <param name="startOffset">The read offset, relative to block head</param>
/// <param name="len">The number of bytes to read</param>
/// <param name="bufferSize">The IO buffer size (not the client buffer size)</param>
/// <param name="verifyChecksum">Whether to verify checksum</param>
/// <param name="clientName">Client name</param>
/// <returns>New BlockReader instance, or null on error.</returns>
/// <exception cref="System.IO.IOException"/>
public static Org.Apache.Hadoop.Hdfs.RemoteBlockReader NewBlockReader(string file
, ExtendedBlock block, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier
> blockToken, long startOffset, long len, int bufferSize, bool verifyChecksum, string
clientName, Peer peer, DatanodeID datanodeID, PeerCache peerCache, CachingStrategy
cachingStrategy)
{
// in and out will be closed when sock is closed (by the caller)
DataOutputStream @out = new DataOutputStream(new BufferedOutputStream(peer.GetOutputStream
()));
new Sender(@out).ReadBlock(block, blockToken, clientName, startOffset, len, verifyChecksum
, cachingStrategy);
//
// Get bytes in block, set streams
//
DataInputStream @in = new DataInputStream(new BufferedInputStream(peer.GetInputStream
(), bufferSize));
DataTransferProtos.BlockOpResponseProto status = DataTransferProtos.BlockOpResponseProto
.ParseFrom(PBHelper.VintPrefixed(@in));
RemoteBlockReader2.CheckSuccess(status, peer, block, file);
DataTransferProtos.ReadOpChecksumInfoProto checksumInfo = status.GetReadOpChecksumInfo
();
DataChecksum checksum = DataTransferProtoUtil.FromProto(checksumInfo.GetChecksum(
));
//Warning when we get CHECKSUM_NULL?
// Read the first chunk offset.
long firstChunkOffset = checksumInfo.GetChunkOffset();
if (firstChunkOffset < 0 || firstChunkOffset > startOffset || firstChunkOffset <=
(startOffset - checksum.GetBytesPerChecksum()))
{
throw new IOException("BlockReader: error in first chunk offset (" + firstChunkOffset
+ ") startOffset is " + startOffset + " for file " + file);
}
return(new Org.Apache.Hadoop.Hdfs.RemoteBlockReader(file, block.GetBlockPoolId(),
block.GetBlockId(), @in, checksum, verifyChecksum, startOffset, firstChunkOffset
, len, peer, datanodeID, peerCache));
}
/// <exception cref="System.IO.IOException"/>
public ChecksumFSOutputSummer(ChecksumFs fs, Path file, EnumSet <CreateFlag> createFlag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
: base(DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, fs.GetBytesPerSum())
)
{
// checksumOpt is passed down to the raw fs. Unless it implements
// checksum impelemts internally, checksumOpt will be ignored.
// If the raw fs does checksum internally, we will end up with
// two layers of checksumming. i.e. checksumming checksum file.
this.datas = fs.GetRawFs().CreateInternal(file, createFlag, absolutePermission, bufferSize
, replication, blockSize, progress, checksumOpt, createParent);
// Now create the chekcsumfile; adjust the buffsize
int bytesPerSum = fs.GetBytesPerSum();
int sumBufferSize = fs.GetSumBufferSize(bytesPerSum, bufferSize);
this.sums = fs.GetRawFs().CreateInternal(fs.GetChecksumFile(file), EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite), absolutePermission, sumBufferSize, replication,
blockSize, progress, checksumOpt, createParent);
sums.Write(ChecksumVersion, 0, ChecksumVersion.Length);
sums.WriteInt(bytesPerSum);
}
public virtual void TestNotMatchedReplicaID()
{
if (Log.IsDebugEnabled())
{
Log.Debug("Running " + GenericTestUtils.GetMethodName());
}
ReplicaInPipelineInterface replicaInfo = dn.data.CreateRbw(StorageType.Default, block
, false).GetReplica();
ReplicaOutputStreams streams = null;
try
{
streams = replicaInfo.CreateStreams(true, DataChecksum.NewDataChecksum(DataChecksum.Type
.Crc32, 512));
streams.GetChecksumOut().Write('a');
dn.data.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock(block, null,
RecoveryId + 1));
try
{
dn.SyncBlock(rBlock, InitBlockRecords(dn));
NUnit.Framework.Assert.Fail("Sync should fail");
}
catch (IOException e)
{
e.Message.StartsWith("Cannot recover ");
}
DatanodeProtocol namenode = dn.GetActiveNamenodeForBP(PoolId);
Org.Mockito.Mockito.Verify(namenode, Org.Mockito.Mockito.Never()).CommitBlockSynchronization
(Matchers.Any <ExtendedBlock>(), Matchers.AnyLong(), Matchers.AnyLong(), Matchers.AnyBoolean
(), Matchers.AnyBoolean(), Matchers.Any <DatanodeID[]>(), Matchers.Any <string[]>(
));
}
finally
{
streams.Close();
}
}
/// <exception cref="System.IO.IOException"/>
internal virtual int AddSomeBlocks(SimulatedFSDataset fsdataset, int startingBlockId
)
{
int bytesAdded = 0;
for (int i = startingBlockId; i < startingBlockId + Numblocks; ++i)
{
ExtendedBlock b = new ExtendedBlock(bpid, i, 0, 0);
// we pass expected len as zero, - fsdataset should use the sizeof actual
// data written
ReplicaInPipelineInterface bInfo = fsdataset.CreateRbw(StorageType.Default, b, false
).GetReplica();
ReplicaOutputStreams @out = bInfo.CreateStreams(true, DataChecksum.NewDataChecksum
(DataChecksum.Type.Crc32, 512));
try
{
OutputStream dataOut = @out.GetDataOut();
NUnit.Framework.Assert.AreEqual(0, fsdataset.GetLength(b));
for (int j = 1; j <= BlockIdToLen(i); ++j)
{
dataOut.Write(j);
NUnit.Framework.Assert.AreEqual(j, bInfo.GetBytesOnDisk());
// correct length even as we write
bytesAdded++;
}
}
finally
{
@out.Close();
}
b.SetNumBytes(BlockIdToLen(i));
fsdataset.FinalizeBlock(b);
NUnit.Framework.Assert.AreEqual(BlockIdToLen(i), fsdataset.GetLength(b));
}
return(bytesAdded);
}
protected internal RemoteBlockReader2(string file, string bpid, long blockId, DataChecksum
checksum, bool verifyChecksum, long startOffset, long firstChunkOffset, long bytesToRead
, Peer peer, DatanodeID datanodeID, PeerCache peerCache)
{
this.isLocal = DFSClient.IsLocalAddress(NetUtils.CreateSocketAddr(datanodeID.GetXferAddr
()));
// Path is used only for printing block and file information in debug
this.peer = peer;
this.datanodeID = datanodeID;
this.@in = peer.GetInputStreamChannel();
this.checksum = checksum;
this.verifyChecksum = verifyChecksum;
this.startOffset = Math.Max(startOffset, 0);
this.filename = file;
this.peerCache = peerCache;
this.blockId = blockId;
// The total number of bytes that we need to transfer from the DN is
// the amount that the user wants (bytesToRead), plus the padding at
// the beginning in order to chunk-align. Note that the DN may elect
// to send more than this amount if the read starts/ends mid-chunk.
this.bytesNeededToFinish = bytesToRead + (startOffset - firstChunkOffset);
bytesPerChecksum = this.checksum.GetBytesPerChecksum();
checksumSize = this.checksum.GetChecksumSize();
}
/// <exception cref="System.IO.IOException"/>
public override void WriteBlock(ExtendedBlock blk, StorageType storageType, Org.Apache.Hadoop.Security.Token.Token
<BlockTokenIdentifier> blockToken, string clientName, DatanodeInfo[] targets, StorageType
[] targetStorageTypes, DatanodeInfo source, BlockConstructionStage stage, int pipelineSize
, long minBytesRcvd, long maxBytesRcvd, long latestGenerationStamp, DataChecksum
requestedChecksum, CachingStrategy cachingStrategy, bool allowLazyPersist, bool
pinning, bool[] targetPinnings)
{
DataTransferProtos.ClientOperationHeaderProto header = DataTransferProtoUtil.BuildClientHeader
(blk, clientName, blockToken);
DataTransferProtos.ChecksumProto checksumProto = DataTransferProtoUtil.ToProto(requestedChecksum
);
DataTransferProtos.OpWriteBlockProto.Builder proto = DataTransferProtos.OpWriteBlockProto
.NewBuilder().SetHeader(header).SetStorageType(PBHelper.ConvertStorageType(storageType
)).AddAllTargets(PBHelper.Convert(targets, 1)).AddAllTargetStorageTypes(PBHelper
.ConvertStorageTypes(targetStorageTypes, 1)).SetStage(DataTransferProtoUtil.ToProto
(stage)).SetPipelineSize(pipelineSize).SetMinBytesRcvd(minBytesRcvd).SetMaxBytesRcvd
(maxBytesRcvd).SetLatestGenerationStamp(latestGenerationStamp).SetRequestedChecksum
(checksumProto).SetCachingStrategy(GetCachingStrategy(cachingStrategy)).SetAllowLazyPersist
(allowLazyPersist).SetPinning(pinning).AddAllTargetPinnings(PBHelper.Convert(targetPinnings
, 1));
if (source != null)
{
proto.SetSource(PBHelper.ConvertDatanodeInfo(source));
}
Send(@out, OP.WriteBlock, ((DataTransferProtos.OpWriteBlockProto)proto.Build()));
}
/// <exception cref="System.IO.IOException"/>
public virtual ReplicaOutputStreams CreateStreams(bool isCreate, DataChecksum requestedChecksum
)
{
return(new ReplicaOutputStreams(null, null, requestedChecksum, false));
}
/// <exception cref="System.IO.IOException"/>
internal virtual void WriteBlock(string poolId, long blockId, DataChecksum checksum
)
{
WriteBlock(new ExtendedBlock(poolId, blockId), BlockConstructionStage.PipelineSetupCreate
, 0L, checksum);
}
/// <summary>
/// Create a checksum output stream that writes
/// the bytes to the given stream.
/// </summary>
/// <param name="out"/>
public IFileOutputStream(OutputStream @out)
: base(@out)
{
sum = DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, int.MaxValue);
barray = new byte[sum.GetChecksumSize()];
}
public virtual void TestDataTransferProtocol()
{
Random random = new Random();
int oneMil = 1024 * 1024;
Path file = new Path("dataprotocol.dat");
int numDataNodes = 1;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, numDataNodes);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes
).Build();
try
{
cluster.WaitActive();
datanode = cluster.GetFileSystem().GetDataNodeStats(HdfsConstants.DatanodeReportType
.Live)[0];
dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr());
FileSystem fileSys = cluster.GetFileSystem();
int fileLen = Math.Min(conf.GetInt(DFSConfigKeys.DfsBlockSizeKey, 4096), 4096);
CreateFile(fileSys, file, fileLen);
// get the first blockid for the file
ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file);
string poolId = firstBlock.GetBlockPoolId();
long newBlockId = firstBlock.GetBlockId() + 1;
recvBuf.Reset();
sendBuf.Reset();
// bad version
recvOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
sendOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1));
SendRecvData("Wrong Version", true);
// bad ops
sendBuf.Reset();
sendOut.WriteShort((short)DataTransferProtocol.DataTransferVersion);
sendOut.WriteByte(OP.WriteBlock.code - 1);
SendRecvData("Wrong Op Code", true);
/* Test OP_WRITE_BLOCK */
sendBuf.Reset();
DataChecksum badChecksum = Org.Mockito.Mockito.Spy(DefaultChecksum);
Org.Mockito.Mockito.DoReturn(-1).When(badChecksum).GetBytesPerChecksum();
WriteBlock(poolId, newBlockId, badChecksum);
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, null, recvOut);
SendRecvData("wrong bytesPerChecksum while writing", true);
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
PacketHeader hdr = new PacketHeader(4, 0, 100, false, -1 - random.Next(oneMil), false
);
// size of packet
// offset in block,
// seqno
// last packet
// bad datalen
hdr.Write(sendOut);
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Error) }).Write(recvOut);
SendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, true);
// test for writing a valid zero size block
sendBuf.Reset();
recvBuf.Reset();
WriteBlock(poolId, ++newBlockId, DefaultChecksum);
hdr = new PacketHeader(8, 0, 100, true, 0, false);
// size of packet
// OffsetInBlock
// sequencenumber
// lastPacketInBlock
// chunk length
hdr.Write(sendOut);
sendOut.WriteInt(0);
// zero checksum
sendOut.Flush();
//ok finally write a block with 0 len
SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut);
new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled
, DataTransferProtos.Status.Success) }).Write(recvOut);
SendRecvData("Writing a zero len block blockid " + newBlockId, false);
/* Test OP_READ_BLOCK */
string bpid = cluster.GetNamesystem().GetBlockPoolId();
ExtendedBlock blk = new ExtendedBlock(bpid, firstBlock.GetLocalBlock());
long blkid = blk.GetBlockId();
// bad block id
sendBuf.Reset();
recvBuf.Reset();
blk.SetBlockId(blkid - 1);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong block ID " + newBlockId + " for read", false);
// negative block start offset -1L
sendBuf.Reset();
blk.SetBlockId(blkid);
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", -1L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative start-offset for read for block " + firstBlock.GetBlockId(
), false);
// bad block start offset
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", fileLen, fileLen,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong start-offset for reading block " + firstBlock.GetBlockId(), false
);
// negative length is ok. Datanode assumes we want to read the whole block.
recvBuf.Reset();
((DataTransferProtos.BlockOpResponseProto)DataTransferProtos.BlockOpResponseProto
.NewBuilder().SetStatus(DataTransferProtos.Status.Success).SetReadOpChecksumInfo
(DataTransferProtos.ReadOpChecksumInfoProto.NewBuilder().SetChecksum(DataTransferProtoUtil
.ToProto(DefaultChecksum)).SetChunkOffset(0L)).Build()).WriteDelimitedTo(recvOut
);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, -1L - random.
Next(oneMil), true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Negative length for reading block " + firstBlock.GetBlockId(), false
);
// length is more than size of block.
recvBuf.Reset();
SendResponse(DataTransferProtos.Status.Error, null, "opReadBlock " + firstBlock +
" received exception java.io.IOException: " + "Offset 0 and length 4097 don't match block "
+ firstBlock + " ( blockLen 4096 )", recvOut);
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen + 1,
true, CachingStrategy.NewDefaultStrategy());
SendRecvData("Wrong length for reading block " + firstBlock.GetBlockId(), false);
//At the end of all this, read the file to make sure that succeeds finally.
sendBuf.Reset();
sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true
, CachingStrategy.NewDefaultStrategy());
ReadFile(fileSys, file, fileLen);
}
finally
{
cluster.Shutdown();
}
}