public virtual void TestThrottler()
{
Configuration conf = new HdfsConfiguration();
FileSystem.SetDefaultUri(conf, "hdfs://localhost:0");
long bandwidthPerSec = 1024 * 1024L;
long TotalBytes = 6 * bandwidthPerSec;
long bytesToSend = TotalBytes;
long start = Time.MonotonicNow();
DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
long totalBytes = 0L;
long bytesSent = 1024 * 512L;
// 0.5MB
throttler.Throttle(bytesSent);
bytesToSend -= bytesSent;
bytesSent = 1024 * 768L;
// 0.75MB
throttler.Throttle(bytesSent);
bytesToSend -= bytesSent;
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
throttler.Throttle(bytesToSend);
long end = Time.MonotonicNow();
NUnit.Framework.Assert.IsTrue(totalBytes * 1000 / (end - start) <= bandwidthPerSec
);
}
/// <exception cref="System.IO.IOException"/>
private static void CopyFileToStream(OutputStream @out, FilePath localfile, FileInputStream
infile, DataTransferThrottler throttler, Canceler canceler)
{
byte[] buf = new byte[HdfsConstants.IoFileBufferSize];
try
{
CheckpointFaultInjector.GetInstance().AboutToSendFile(localfile);
if (CheckpointFaultInjector.GetInstance().ShouldSendShortFile(localfile))
{
// Test sending image shorter than localfile
long len = localfile.Length();
buf = new byte[(int)Math.Min(len / 2, HdfsConstants.IoFileBufferSize)];
// This will read at most half of the image
// and the rest of the image will be sent over the wire
infile.Read(buf);
}
int num = 1;
while (num > 0)
{
if (canceler != null && canceler.IsCancelled())
{
throw new SaveNamespaceCancelledException(canceler.GetCancellationReason());
}
num = infile.Read(buf);
if (num <= 0)
{
break;
}
if (CheckpointFaultInjector.GetInstance().ShouldCorruptAByte(localfile))
{
// Simulate a corrupted byte on the wire
Log.Warn("SIMULATING A CORRUPT BYTE IN IMAGE TRANSFER!");
buf[0]++;
}
@out.Write(buf, 0, num);
if (throttler != null)
{
throttler.Throttle(num, canceler);
}
}
}
catch (EofException)
{
Log.Info("Connection closed by client");
@out = null;
}
finally
{
// so we don't close in the finally
if (@out != null)
{
@out.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
private static MD5Hash ReceiveFile(string url, IList <FilePath> localPaths, Storage
dstStorage, bool getChecksum, long advertisedSize, MD5Hash advertisedDigest, string
fsImageName, InputStream stream, DataTransferThrottler throttler)
{
long startTime = Time.MonotonicNow();
if (localPaths != null)
{
// If the local paths refer to directories, use the server-provided header
// as the filename within that directory
IList <FilePath> newLocalPaths = new AList <FilePath>();
foreach (FilePath localPath in localPaths)
{
if (localPath.IsDirectory())
{
if (fsImageName == null)
{
throw new IOException("No filename header provided by server");
}
newLocalPaths.AddItem(new FilePath(localPath, fsImageName));
}
else
{
newLocalPaths.AddItem(localPath);
}
}
localPaths = newLocalPaths;
}
long received = 0;
MessageDigest digester = null;
if (getChecksum)
{
digester = MD5Hash.GetDigester();
stream = new DigestInputStream(stream, digester);
}
bool finishedReceiving = false;
IList <FileOutputStream> outputStreams = Lists.NewArrayList();
try
{
if (localPaths != null)
{
foreach (FilePath f in localPaths)
{
try
{
if (f.Exists())
{
Log.Warn("Overwriting existing file " + f + " with file downloaded from " + url);
}
outputStreams.AddItem(new FileOutputStream(f));
}
catch (IOException ioe)
{
Log.Warn("Unable to download file " + f, ioe);
// This will be null if we're downloading the fsimage to a file
// outside of an NNStorage directory.
if (dstStorage != null && (dstStorage is StorageErrorReporter))
{
((StorageErrorReporter)dstStorage).ReportErrorOnFile(f);
}
}
}
if (outputStreams.IsEmpty())
{
throw new IOException("Unable to download to any storage directory");
}
}
int num = 1;
byte[] buf = new byte[HdfsConstants.IoFileBufferSize];
while (num > 0)
{
num = stream.Read(buf);
if (num > 0)
{
received += num;
foreach (FileOutputStream fos in outputStreams)
{
fos.Write(buf, 0, num);
}
if (throttler != null)
{
throttler.Throttle(num);
}
}
}
finishedReceiving = true;
}
finally
{
stream.Close();
foreach (FileOutputStream fos in outputStreams)
{
fos.GetChannel().Force(true);
fos.Close();
}
// Something went wrong and did not finish reading.
// Remove the temporary files.
if (!finishedReceiving)
{
DeleteTmpFiles(localPaths);
}
if (finishedReceiving && received != advertisedSize)
{
// only throw this exception if we think we read all of it on our end
// -- otherwise a client-side IOException would be masked by this
// exception that makes it look like a server-side problem!
DeleteTmpFiles(localPaths);
throw new IOException("File " + url + " received length " + received + " is not of the advertised size "
+ advertisedSize);
}
}
double xferSec = Math.Max(((float)(Time.MonotonicNow() - startTime)) / 1000.0, 0.001
);
long xferKb = received / 1024;
Log.Info(string.Format("Transfer took %.2fs at %.2f KB/s", xferSec, xferKb / xferSec
));
if (digester != null)
{
MD5Hash computedDigest = new MD5Hash(digester.Digest());
if (advertisedDigest != null && !computedDigest.Equals(advertisedDigest))
{
DeleteTmpFiles(localPaths);
throw new IOException("File " + url + " computed digest " + computedDigest + " does not match advertised digest "
+ advertisedDigest);
}
return(computedDigest);
}
else
{
return(null);
}
}
/// <summary>Sends a packet with up to maxChunks chunks of data.</summary>
/// <param name="pkt">buffer used for writing packet data</param>
/// <param name="maxChunks">maximum number of chunks to send</param>
/// <param name="out">stream to send data to</param>
/// <param name="transferTo">use transferTo to send data</param>
/// <param name="throttler">used for throttling data transfer bandwidth</param>
/// <exception cref="System.IO.IOException"/>
private int SendPacket(ByteBuffer pkt, int maxChunks, OutputStream @out, bool transferTo
, DataTransferThrottler throttler)
{
int dataLen = (int)Math.Min(endOffset - offset, (chunkSize * (long)maxChunks));
int numChunks = NumberOfChunks(dataLen);
// Number of chunks be sent in the packet
int checksumDataLen = numChunks * checksumSize;
int packetLen = dataLen + checksumDataLen + 4;
bool lastDataPacket = offset + dataLen == endOffset && dataLen > 0;
// The packet buffer is organized as follows:
// _______HHHHCCCCD?D?D?D?
// ^ ^
// | \ checksumOff
// \ headerOff
// _ padding, since the header is variable-length
// H = header and length prefixes
// C = checksums
// D? = data, if transferTo is false.
int headerLen = WritePacketHeader(pkt, dataLen, packetLen);
// Per above, the header doesn't start at the beginning of the
// buffer
int headerOff = pkt.Position() - headerLen;
int checksumOff = pkt.Position();
byte[] buf = ((byte[])pkt.Array());
if (checksumSize > 0 && checksumIn != null)
{
ReadChecksum(buf, checksumOff, checksumDataLen);
// write in progress that we need to use to get last checksum
if (lastDataPacket && lastChunkChecksum != null)
{
int start = checksumOff + checksumDataLen - checksumSize;
byte[] updatedChecksum = lastChunkChecksum.GetChecksum();
if (updatedChecksum != null)
{
System.Array.Copy(updatedChecksum, 0, buf, start, checksumSize);
}
}
}
int dataOff = checksumOff + checksumDataLen;
if (!transferTo)
{
// normal transfer
IOUtils.ReadFully(blockIn, buf, dataOff, dataLen);
if (verifyChecksum)
{
VerifyChecksum(buf, dataOff, dataLen, numChunks, checksumOff);
}
}
try
{
if (transferTo)
{
SocketOutputStream sockOut = (SocketOutputStream)@out;
// First write header and checksums
sockOut.Write(buf, headerOff, dataOff - headerOff);
// no need to flush since we know out is not a buffered stream
FileChannel fileCh = ((FileInputStream)blockIn).GetChannel();
LongWritable waitTime = new LongWritable();
LongWritable transferTime = new LongWritable();
sockOut.TransferToFully(fileCh, blockInPosition, dataLen, waitTime, transferTime);
datanode.metrics.AddSendDataPacketBlockedOnNetworkNanos(waitTime.Get());
datanode.metrics.AddSendDataPacketTransferNanos(transferTime.Get());
blockInPosition += dataLen;
}
else
{
// normal transfer
@out.Write(buf, headerOff, dataOff + dataLen - headerOff);
}
}
catch (IOException e)
{
if (e is SocketTimeoutException)
{
}
else
{
/*
* writing to client timed out. This happens if the client reads
* part of a block and then decides not to read the rest (but leaves
* the socket open).
*
* Reporting of this case is done in DataXceiver#run
*/
/* Exception while writing to the client. Connection closure from
* the other end is mostly the case and we do not care much about
* it. But other things can go wrong, especially in transferTo(),
* which we do not want to ignore.
*
* The message parsing below should not be considered as a good
* coding example. NEVER do it to drive a program logic. NEVER.
* It was done here because the NIO throws an IOException for EPIPE.
*/
string ioem = e.Message;
if (!ioem.StartsWith("Broken pipe") && !ioem.StartsWith("Connection reset"))
{
Log.Error("BlockSender.sendChunks() exception: ", e);
}
datanode.GetBlockScanner().MarkSuspectBlock(volumeRef.GetVolume().GetStorageID(),
block);
}
throw IoeToSocketException(e);
}
if (throttler != null)
{
// rebalancing so throttle
throttler.Throttle(packetLen);
}
return(dataLen);
}
