/// <exception cref="Javax.Servlet.ServletException"/> /// <exception cref="System.IO.IOException"/> protected override void DoGet(HttpServletRequest request, HttpServletResponse response ) { FileInputStream editFileIn = null; try { ServletContext context = GetServletContext(); Configuration conf = (Configuration)GetServletContext().GetAttribute(JspHelper.CurrentConf ); string journalId = request.GetParameter(JournalIdParam); QuorumJournalManager.CheckJournalId(journalId); JNStorage storage = JournalNodeHttpServer.GetJournalFromContext(context, journalId ).GetStorage(); // Check security if (!CheckRequestorOrSendError(conf, request, response)) { return; } // Check that the namespace info is correct if (!CheckStorageInfoOrSendError(storage, request, response)) { return; } long segmentTxId = ServletUtil.ParseLongParam(request, SegmentTxidParam); FileJournalManager fjm = storage.GetJournalManager(); FilePath editFile; lock (fjm) { // Synchronize on the FJM so that the file doesn't get finalized // out from underneath us while we're in the process of opening // it up. FileJournalManager.EditLogFile elf = fjm.GetLogFile(segmentTxId); if (elf == null) { response.SendError(HttpServletResponse.ScNotFound, "No edit log found starting at txid " + segmentTxId); return; } editFile = elf.GetFile(); ImageServlet.SetVerificationHeadersForGet(response, editFile); ImageServlet.SetFileNameHeaders(response, editFile); editFileIn = new FileInputStream(editFile); } DataTransferThrottler throttler = ImageServlet.GetThrottler(conf); // send edits TransferFsImage.CopyFileToStream(response.GetOutputStream(), editFile, editFileIn , throttler); } catch (Exception t) { string errMsg = "getedit failed. " + StringUtils.StringifyException(t); response.SendError(HttpServletResponse.ScInternalServerError, errMsg); throw new IOException(errMsg); } finally { IOUtils.CloseStream(editFileIn); } }
public virtual void TestThrottler() { Configuration conf = new HdfsConfiguration(); FileSystem.SetDefaultUri(conf, "hdfs://localhost:0"); long bandwidthPerSec = 1024 * 1024L; long TotalBytes = 6 * bandwidthPerSec; long bytesToSend = TotalBytes; long start = Time.MonotonicNow(); DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec); long totalBytes = 0L; long bytesSent = 1024 * 512L; // 0.5MB throttler.Throttle(bytesSent); bytesToSend -= bytesSent; bytesSent = 1024 * 768L; // 0.75MB throttler.Throttle(bytesSent); bytesToSend -= bytesSent; try { Sharpen.Thread.Sleep(1000); } catch (Exception) { } throttler.Throttle(bytesToSend); long end = Time.MonotonicNow(); NUnit.Framework.Assert.IsTrue(totalBytes * 1000 / (end - start) <= bandwidthPerSec ); }
/// <exception cref="System.IO.IOException"/> private static void CopyFileToStream(OutputStream @out, FilePath localfile, FileInputStream infile, DataTransferThrottler throttler, Canceler canceler) { byte[] buf = new byte[HdfsConstants.IoFileBufferSize]; try { CheckpointFaultInjector.GetInstance().AboutToSendFile(localfile); if (CheckpointFaultInjector.GetInstance().ShouldSendShortFile(localfile)) { // Test sending image shorter than localfile long len = localfile.Length(); buf = new byte[(int)Math.Min(len / 2, HdfsConstants.IoFileBufferSize)]; // This will read at most half of the image // and the rest of the image will be sent over the wire infile.Read(buf); } int num = 1; while (num > 0) { if (canceler != null && canceler.IsCancelled()) { throw new SaveNamespaceCancelledException(canceler.GetCancellationReason()); } num = infile.Read(buf); if (num <= 0) { break; } if (CheckpointFaultInjector.GetInstance().ShouldCorruptAByte(localfile)) { // Simulate a corrupted byte on the wire Log.Warn("SIMULATING A CORRUPT BYTE IN IMAGE TRANSFER!"); buf[0]++; } @out.Write(buf, 0, num); if (throttler != null) { throttler.Throttle(num, canceler); } } } catch (EofException) { Log.Info("Connection closed by client"); @out = null; } finally { // so we don't close in the finally if (@out != null) { @out.Close(); } } }
/// <summary>Construct a throttler from conf</summary> /// <param name="conf">configuration</param> /// <returns>a data transfer throttler</returns> public static DataTransferThrottler GetThrottler(Configuration conf) { long transferBandwidth = conf.GetLong(DFSConfigKeys.DfsImageTransferRateKey, DFSConfigKeys .DfsImageTransferRateDefault); DataTransferThrottler throttler = null; if (transferBandwidth > 0) { throttler = new DataTransferThrottler(transferBandwidth); } return(throttler); }
/// <exception cref="System.IO.IOException"/> internal static MD5Hash HandleUploadImageRequest(HttpServletRequest request, long imageTxId, Storage dstStorage, InputStream stream, long advertisedSize, DataTransferThrottler throttler) { string fileName = NNStorage.GetCheckpointImageFileName(imageTxId); IList <FilePath> dstFiles = dstStorage.GetFiles(NNStorage.NameNodeDirType.Image, fileName ); if (dstFiles.IsEmpty()) { throw new IOException("No targets in destination storage!"); } MD5Hash advertisedDigest = ParseMD5Header(request); MD5Hash hash = ReceiveFile(fileName, dstFiles, dstStorage, true, advertisedSize, advertisedDigest, fileName, stream, throttler); Log.Info("Downloaded file " + dstFiles[0].GetName() + " size " + dstFiles[0].Length () + " bytes."); return(hash); }
/// <exception cref="System.IO.IOException"/> private static MD5Hash ReceiveFile(string url, IList <FilePath> localPaths, Storage dstStorage, bool getChecksum, long advertisedSize, MD5Hash advertisedDigest, string fsImageName, InputStream stream, DataTransferThrottler throttler) { long startTime = Time.MonotonicNow(); if (localPaths != null) { // If the local paths refer to directories, use the server-provided header // as the filename within that directory IList <FilePath> newLocalPaths = new AList <FilePath>(); foreach (FilePath localPath in localPaths) { if (localPath.IsDirectory()) { if (fsImageName == null) { throw new IOException("No filename header provided by server"); } newLocalPaths.AddItem(new FilePath(localPath, fsImageName)); } else { newLocalPaths.AddItem(localPath); } } localPaths = newLocalPaths; } long received = 0; MessageDigest digester = null; if (getChecksum) { digester = MD5Hash.GetDigester(); stream = new DigestInputStream(stream, digester); } bool finishedReceiving = false; IList <FileOutputStream> outputStreams = Lists.NewArrayList(); try { if (localPaths != null) { foreach (FilePath f in localPaths) { try { if (f.Exists()) { Log.Warn("Overwriting existing file " + f + " with file downloaded from " + url); } outputStreams.AddItem(new FileOutputStream(f)); } catch (IOException ioe) { Log.Warn("Unable to download file " + f, ioe); // This will be null if we're downloading the fsimage to a file // outside of an NNStorage directory. if (dstStorage != null && (dstStorage is StorageErrorReporter)) { ((StorageErrorReporter)dstStorage).ReportErrorOnFile(f); } } } if (outputStreams.IsEmpty()) { throw new IOException("Unable to download to any storage directory"); } } int num = 1; byte[] buf = new byte[HdfsConstants.IoFileBufferSize]; while (num > 0) { num = stream.Read(buf); if (num > 0) { received += num; foreach (FileOutputStream fos in outputStreams) { fos.Write(buf, 0, num); } if (throttler != null) { throttler.Throttle(num); } } } finishedReceiving = true; } finally { stream.Close(); foreach (FileOutputStream fos in outputStreams) { fos.GetChannel().Force(true); fos.Close(); } // Something went wrong and did not finish reading. // Remove the temporary files. if (!finishedReceiving) { DeleteTmpFiles(localPaths); } if (finishedReceiving && received != advertisedSize) { // only throw this exception if we think we read all of it on our end // -- otherwise a client-side IOException would be masked by this // exception that makes it look like a server-side problem! DeleteTmpFiles(localPaths); throw new IOException("File " + url + " received length " + received + " is not of the advertised size " + advertisedSize); } } double xferSec = Math.Max(((float)(Time.MonotonicNow() - startTime)) / 1000.0, 0.001 ); long xferKb = received / 1024; Log.Info(string.Format("Transfer took %.2fs at %.2f KB/s", xferSec, xferKb / xferSec )); if (digester != null) { MD5Hash computedDigest = new MD5Hash(digester.Digest()); if (advertisedDigest != null && !computedDigest.Equals(advertisedDigest)) { DeleteTmpFiles(localPaths); throw new IOException("File " + url + " computed digest " + computedDigest + " does not match advertised digest " + advertisedDigest); } return(computedDigest); } else { return(null); } }
/// <summary> /// A server-side method to respond to a getfile http request /// Copies the contents of the local file into the output stream. /// </summary> /// <exception cref="System.IO.IOException"/> public static void CopyFileToStream(OutputStream @out, FilePath localfile, FileInputStream infile, DataTransferThrottler throttler) { CopyFileToStream(@out, localfile, infile, throttler, null); }
/// <exception cref="System.IO.IOException"/> private long DoSendBlock(DataOutputStream @out, OutputStream baseStream, DataTransferThrottler throttler) { if (@out == null) { throw new IOException("out stream is null"); } initialOffset = offset; long totalRead = 0; OutputStream streamForSendChunks = @out; lastCacheDropOffset = initialOffset; if (IsLongRead() && blockInFd != null) { // Advise that this file descriptor will be accessed sequentially. NativeIO.POSIX.GetCacheManipulator().PosixFadviseIfPossible(block.GetBlockName(), blockInFd, 0, 0, NativeIO.POSIX.PosixFadvSequential); } // Trigger readahead of beginning of file if configured. ManageOsCache(); long startTime = ClientTraceLog.IsDebugEnabled() ? Runtime.NanoTime() : 0; try { int maxChunksPerPacket; int pktBufSize = PacketHeader.PktMaxHeaderLen; bool transferTo = transferToAllowed && !verifyChecksum && baseStream is SocketOutputStream && blockIn is FileInputStream; if (transferTo) { FileChannel fileChannel = ((FileInputStream)blockIn).GetChannel(); blockInPosition = fileChannel.Position(); streamForSendChunks = baseStream; maxChunksPerPacket = NumberOfChunks(TransfertoBufferSize); // Smaller packet size to only hold checksum when doing transferTo pktBufSize += checksumSize * maxChunksPerPacket; } else { maxChunksPerPacket = Math.Max(1, NumberOfChunks(HdfsConstants.IoFileBufferSize)); // Packet size includes both checksum and data pktBufSize += (chunkSize + checksumSize) * maxChunksPerPacket; } ByteBuffer pktBuf = ByteBuffer.Allocate(pktBufSize); while (endOffset > offset && !Sharpen.Thread.CurrentThread().IsInterrupted()) { ManageOsCache(); long len = SendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo , throttler); offset += len; totalRead += len + (NumberOfChunks(len) * checksumSize); seqno++; } // If this thread was interrupted, then it did not send the full block. if (!Sharpen.Thread.CurrentThread().IsInterrupted()) { try { // send an empty packet to mark the end of the block SendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo, throttler ); @out.Flush(); } catch (IOException e) { //socket error throw IoeToSocketException(e); } sentEntireByteRange = true; } } finally { if ((clientTraceFmt != null) && ClientTraceLog.IsDebugEnabled()) { long endTime = Runtime.NanoTime(); ClientTraceLog.Debug(string.Format(clientTraceFmt, totalRead, initialOffset, endTime - startTime)); } Close(); } return(totalRead); }
/// <summary> /// sendBlock() is used to read block and its metadata and stream the data to /// either a client or to another datanode. /// </summary> /// <param name="out">stream to which the block is written to</param> /// <param name="baseStream"> /// optional. if non-null, <code>out</code> is assumed to /// be a wrapper over this stream. This enables optimizations for /// sending the data, e.g. /// <see cref="Org.Apache.Hadoop.Net.SocketOutputStream.TransferToFully(Sharpen.FileChannel, long, int) /// "/> /// . /// </param> /// <param name="throttler">for sending data.</param> /// <returns>total bytes read, including checksum data.</returns> /// <exception cref="System.IO.IOException"/> internal virtual long SendBlock(DataOutputStream @out, OutputStream baseStream, DataTransferThrottler throttler) { TraceScope scope = Trace.StartSpan("sendBlock_" + block.GetBlockId(), Sampler.Never ); try { return(DoSendBlock(@out, baseStream, throttler)); } finally { scope.Close(); } }
/// <summary>Sends a packet with up to maxChunks chunks of data.</summary> /// <param name="pkt">buffer used for writing packet data</param> /// <param name="maxChunks">maximum number of chunks to send</param> /// <param name="out">stream to send data to</param> /// <param name="transferTo">use transferTo to send data</param> /// <param name="throttler">used for throttling data transfer bandwidth</param> /// <exception cref="System.IO.IOException"/> private int SendPacket(ByteBuffer pkt, int maxChunks, OutputStream @out, bool transferTo , DataTransferThrottler throttler) { int dataLen = (int)Math.Min(endOffset - offset, (chunkSize * (long)maxChunks)); int numChunks = NumberOfChunks(dataLen); // Number of chunks be sent in the packet int checksumDataLen = numChunks * checksumSize; int packetLen = dataLen + checksumDataLen + 4; bool lastDataPacket = offset + dataLen == endOffset && dataLen > 0; // The packet buffer is organized as follows: // _______HHHHCCCCD?D?D?D? // ^ ^ // | \ checksumOff // \ headerOff // _ padding, since the header is variable-length // H = header and length prefixes // C = checksums // D? = data, if transferTo is false. int headerLen = WritePacketHeader(pkt, dataLen, packetLen); // Per above, the header doesn't start at the beginning of the // buffer int headerOff = pkt.Position() - headerLen; int checksumOff = pkt.Position(); byte[] buf = ((byte[])pkt.Array()); if (checksumSize > 0 && checksumIn != null) { ReadChecksum(buf, checksumOff, checksumDataLen); // write in progress that we need to use to get last checksum if (lastDataPacket && lastChunkChecksum != null) { int start = checksumOff + checksumDataLen - checksumSize; byte[] updatedChecksum = lastChunkChecksum.GetChecksum(); if (updatedChecksum != null) { System.Array.Copy(updatedChecksum, 0, buf, start, checksumSize); } } } int dataOff = checksumOff + checksumDataLen; if (!transferTo) { // normal transfer IOUtils.ReadFully(blockIn, buf, dataOff, dataLen); if (verifyChecksum) { VerifyChecksum(buf, dataOff, dataLen, numChunks, checksumOff); } } try { if (transferTo) { SocketOutputStream sockOut = (SocketOutputStream)@out; // First write header and checksums sockOut.Write(buf, headerOff, dataOff - headerOff); // no need to flush since we know out is not a buffered stream FileChannel fileCh = ((FileInputStream)blockIn).GetChannel(); LongWritable waitTime = new LongWritable(); LongWritable transferTime = new LongWritable(); sockOut.TransferToFully(fileCh, blockInPosition, dataLen, waitTime, transferTime); datanode.metrics.AddSendDataPacketBlockedOnNetworkNanos(waitTime.Get()); datanode.metrics.AddSendDataPacketTransferNanos(transferTime.Get()); blockInPosition += dataLen; } else { // normal transfer @out.Write(buf, headerOff, dataOff + dataLen - headerOff); } } catch (IOException e) { if (e is SocketTimeoutException) { } else { /* * writing to client timed out. This happens if the client reads * part of a block and then decides not to read the rest (but leaves * the socket open). * * Reporting of this case is done in DataXceiver#run */ /* Exception while writing to the client. Connection closure from * the other end is mostly the case and we do not care much about * it. But other things can go wrong, especially in transferTo(), * which we do not want to ignore. * * The message parsing below should not be considered as a good * coding example. NEVER do it to drive a program logic. NEVER. * It was done here because the NIO throws an IOException for EPIPE. */ string ioem = e.Message; if (!ioem.StartsWith("Broken pipe") && !ioem.StartsWith("Connection reset")) { Log.Error("BlockSender.sendChunks() exception: ", e); } datanode.GetBlockScanner().MarkSuspectBlock(volumeRef.GetVolume().GetStorageID(), block); } throw IoeToSocketException(e); } if (throttler != null) { // rebalancing so throttle throttler.Throttle(packetLen); } return(dataLen); }