/// <summary>
/// Returns the next batch of events in the stream, waiting indefinitely if
/// a new batch is not immediately available.
/// </summary>
/// <exception cref="System.IO.IOException">
/// see
/// <see cref="Poll()"/>
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException">
/// see
/// <see cref="Poll()"/>
/// </exception>
/// <exception cref="System.Exception">if the calling thread is interrupted</exception>
public virtual EventBatch Take()
{
TraceScope scope = Trace.StartSpan("inotifyTake", traceSampler);
EventBatch next = null;
try
{
int nextWaitMin = InitialWaitMs;
while ((next = Poll()) == null)
{
// sleep for a random period between nextWaitMin and nextWaitMin * 2
// to avoid stampedes at the NN if there are multiple clients
int sleepTime = nextWaitMin + rng.Next(nextWaitMin);
Log.Debug("take(): poll() returned null, sleeping for {} ms", sleepTime);
Sharpen.Thread.Sleep(sleepTime);
// the maximum sleep is 2 minutes
nextWaitMin = Math.Min(60000, nextWaitMin * 2);
}
}
finally
{
scope.Close();
}
return(next);
}
/// <summary>Receive OP_WRITE_BLOCK</summary>
/// <exception cref="System.IO.IOException"/>
private void OpWriteBlock(DataInputStream @in)
{
DataTransferProtos.OpWriteBlockProto proto = DataTransferProtos.OpWriteBlockProto
.ParseFrom(PBHelper.VintPrefixed(@in));
DatanodeInfo[] targets = PBHelper.Convert(proto.GetTargetsList());
TraceScope traceScope = DataTransferProtoUtil.ContinueTraceSpan(proto.GetHeader()
, proto.GetType().Name);
try
{
WriteBlock(PBHelper.Convert(proto.GetHeader().GetBaseHeader().GetBlock()), PBHelper
.ConvertStorageType(proto.GetStorageType()), PBHelper.Convert(proto.GetHeader().
GetBaseHeader().GetToken()), proto.GetHeader().GetClientName(), targets, PBHelper
.ConvertStorageTypes(proto.GetTargetStorageTypesList(), targets.Length), PBHelper
.Convert(proto.GetSource()), DataTransferProtoUtil.FromProto(proto.GetStage()),
proto.GetPipelineSize(), proto.GetMinBytesRcvd(), proto.GetMaxBytesRcvd(), proto
.GetLatestGenerationStamp(), DataTransferProtoUtil.FromProto(proto.GetRequestedChecksum
()), (proto.HasCachingStrategy() ? GetCachingStrategy(proto.GetCachingStrategy()
) : CachingStrategy.NewDefaultStrategy()), (proto.HasAllowLazyPersist() ? proto.
GetAllowLazyPersist() : false), (proto.HasPinning() ? proto.GetPinning() : false
), (PBHelper.ConvertBooleanList(proto.GetTargetPinningsList())));
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
}
/// <exception cref="System.Exception"/>
public virtual HdfsBlocksMetadata Call()
{
HdfsBlocksMetadata metadata = null;
// Create the RPC proxy and make the RPC
ClientDatanodeProtocol cdp = null;
TraceScope scope = Trace.StartSpan("getHdfsBlocksMetadata", parentSpan);
try
{
cdp = DFSUtil.CreateClientDatanodeProtocolProxy(datanode, configuration, timeout,
connectToDnViaHostname);
metadata = cdp.GetHdfsBlocksMetadata(poolId, blockIds, dnTokens);
}
catch (IOException e)
{
// Bubble this up to the caller, handle with the Future
throw;
}
finally
{
scope.Close();
if (cdp != null)
{
RPC.StopProxy(cdp);
}
}
return(metadata);
}
/// <exception cref="System.Exception"/>
public virtual object Invoke(object proxy, MethodInfo method, object[] args)
{
long startTime = 0;
if (Log.IsDebugEnabled())
{
startTime = Time.Now();
}
TraceScope traceScope = null;
if (Trace.IsTracing())
{
traceScope = Trace.StartSpan(RpcClientUtil.MethodToTraceString(method));
}
ObjectWritable value;
try
{
value = (ObjectWritable)client.Call(RPC.RpcKind.RpcWritable, new WritableRpcEngine.Invocation
(method, args), remoteId, fallbackToSimpleAuth);
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
if (Log.IsDebugEnabled())
{
long callTime = Time.Now() - startTime;
Log.Debug("Call: " + method.Name + " " + callTime);
}
return(value.Get());
}
/// <exception cref="System.Exception"/>
public virtual void ReadWithTracing()
{
string fileName = "testReadTraceHooks.dat";
WriteTestFile(fileName);
long startTime = Runtime.CurrentTimeMillis();
TraceScope ts = Trace.StartSpan("testReadTraceHooks", Sampler.Always);
ReadTestFile(fileName);
ts.Close();
long endTime = Runtime.CurrentTimeMillis();
string[] expectedSpanNames = new string[] { "testReadTraceHooks", "org.apache.hadoop.hdfs.protocol.ClientProtocol.getBlockLocations"
, "ClientNamenodeProtocol#getBlockLocations", "OpReadBlockProto" };
AssertSpanNamesFound(expectedSpanNames);
// The trace should last about the same amount of time as the test
IDictionary <string, IList <Span> > map = TestTracing.SetSpanReceiver.SetHolder.GetMap
();
Span s = map["testReadTraceHooks"][0];
NUnit.Framework.Assert.IsNotNull(s);
long spanStart = s.GetStartTimeMillis();
long spanEnd = s.GetStopTimeMillis();
NUnit.Framework.Assert.IsTrue(spanStart - startTime < 100);
NUnit.Framework.Assert.IsTrue(spanEnd - endTime < 100);
// There should only be one trace id as it should all be homed in the
// top trace.
foreach (Span span in TestTracing.SetSpanReceiver.SetHolder.spans.Values)
{
NUnit.Framework.Assert.AreEqual(ts.GetSpan().GetTraceId(), span.GetTraceId());
}
TestTracing.SetSpanReceiver.SetHolder.spans.Clear();
}
/// <exception cref="System.IO.IOException"/>
public virtual int Read(ByteBuffer buf)
{
if (curDataSlice == null || curDataSlice.Remaining() == 0 && bytesNeededToFinish
> 0)
{
TraceScope scope = Trace.StartSpan("RemoteBlockReader2#readNextPacket(" + blockId
+ ")", Sampler.Never);
try
{
ReadNextPacket();
}
finally
{
scope.Close();
}
}
if (curDataSlice.Remaining() == 0)
{
// we're at EOF now
return(-1);
}
int nRead = Math.Min(curDataSlice.Remaining(), buf.Remaining());
ByteBuffer writeSlice = curDataSlice.Duplicate();
writeSlice.Limit(writeSlice.Position() + nRead);
buf.Put(writeSlice);
curDataSlice.Position(writeSlice.Position());
return(nRead);
}
public virtual void TestShortCircuitTraceHooks()
{
Assume.AssumeTrue(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows);
conf = new Configuration();
conf.Set(DFSConfigKeys.DfsClientHtracePrefix + SpanReceiverHost.SpanReceiversConfSuffix
, typeof(TestTracing.SetSpanReceiver).FullName);
conf.SetLong("dfs.blocksize", 100 * 1024);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "testShortCircuitTraceHooks._PORT"
);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
dfs = cluster.GetFileSystem();
try
{
DFSTestUtil.CreateFile(dfs, TestPath, TestLength, (short)1, 5678L);
TraceScope ts = Trace.StartSpan("testShortCircuitTraceHooks", Sampler.Always);
FSDataInputStream stream = dfs.Open(TestPath);
byte[] buf = new byte[TestLength];
IOUtils.ReadFully(stream, buf, 0, TestLength);
stream.Close();
ts.Close();
string[] expectedSpanNames = new string[] { "OpRequestShortCircuitAccessProto", "ShortCircuitShmRequestProto" };
TestTracing.AssertSpanNamesFound(expectedSpanNames);
}
finally
{
dfs.Close();
cluster.Shutdown();
}
}
/// <summary>Reads bytes into a buffer until EOF or the buffer's limit is reached</summary>
/// <exception cref="System.IO.IOException"/>
private int FillBuffer(FileInputStream stream, ByteBuffer buf)
{
TraceScope scope = Trace.StartSpan("BlockReaderLocalLegacy#fillBuffer(" + blockId
+ ")", Sampler.Never);
try
{
int bytesRead = stream.GetChannel().Read(buf);
if (bytesRead < 0)
{
//EOF
return(bytesRead);
}
while (buf.Remaining() > 0)
{
int n = stream.GetChannel().Read(buf);
if (n < 0)
{
//EOF
return(bytesRead);
}
bytesRead += n;
}
return(bytesRead);
}
finally
{
scope.Close();
}
}
/// <summary>
/// Returns the next batch of events in the stream or null if no new
/// batches are currently available.
/// </summary>
/// <exception cref="System.IO.IOException">
/// because of network error or edit log
/// corruption. Also possible if JournalNodes are unresponsive in the
/// QJM setting (even one unresponsive JournalNode is enough in rare cases),
/// so catching this exception and retrying at least a few times is
/// recommended.
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException">
/// if we cannot return the next batch in the
/// stream because the data for the events (and possibly some subsequent
/// events) has been deleted (generally because this stream is a very large
/// number of transactions behind the current state of the NameNode). It is
/// safe to continue reading from the stream after this exception is thrown
/// The next available batch of events will be returned.
/// </exception>
public virtual EventBatch Poll()
{
TraceScope scope = Trace.StartSpan("inotifyPoll", traceSampler);
try
{
// need to keep retrying until the NN sends us the latest committed txid
if (lastReadTxid == -1)
{
Log.Debug("poll(): lastReadTxid is -1, reading current txid from NN");
lastReadTxid = namenode.GetCurrentEditLogTxid();
return(null);
}
if (!it.HasNext())
{
EventBatchList el = namenode.GetEditsFromTxid(lastReadTxid + 1);
if (el.GetLastTxid() != -1)
{
// we only want to set syncTxid when we were actually able to read some
// edits on the NN -- otherwise it will seem like edits are being
// generated faster than we can read them when the problem is really
// that we are temporarily unable to read edits
syncTxid = el.GetSyncTxid();
it = el.GetBatches().GetEnumerator();
long formerLastReadTxid = lastReadTxid;
lastReadTxid = el.GetLastTxid();
if (el.GetFirstTxid() != formerLastReadTxid + 1)
{
throw new MissingEventsException(formerLastReadTxid + 1, el.GetFirstTxid());
}
}
else
{
Log.Debug("poll(): read no edits from the NN when requesting edits " + "after txid {}"
, lastReadTxid);
return(null);
}
}
if (it.HasNext())
{
// can be empty if el.getLastTxid != -1 but none of the
// newly seen edit log ops actually got converted to events
return(it.Next());
}
else
{
return(null);
}
}
finally
{
scope.Close();
}
}
/// <exception cref="System.IO.IOException"/>
public override BatchedRemoteIterator.BatchedEntries <CachePoolEntry> MakeRequest(
string prevKey)
{
TraceScope scope = Trace.StartSpan("listCachePools", traceSampler);
try
{
return(namenode.ListCachePools(prevKey));
}
finally
{
scope.Close();
}
}
/// <exception cref="System.IO.IOException"/>
public override BatchedRemoteIterator.BatchedEntries <EncryptionZone> MakeRequest(
long prevId)
{
TraceScope scope = Trace.StartSpan("listEncryptionZones", traceSampler);
try
{
return(namenode.ListEncryptionZones(prevId));
}
finally
{
scope.Close();
}
}
/// <summary>
/// sendBlock() is used to read block and its metadata and stream the data to
/// either a client or to another datanode.
/// </summary>
/// <param name="out">stream to which the block is written to</param>
/// <param name="baseStream">
/// optional. if non-null, <code>out</code> is assumed to
/// be a wrapper over this stream. This enables optimizations for
/// sending the data, e.g.
/// <see cref="Org.Apache.Hadoop.Net.SocketOutputStream.TransferToFully(Sharpen.FileChannel, long, int)
/// "/>
/// .
/// </param>
/// <param name="throttler">for sending data.</param>
/// <returns>total bytes read, including checksum data.</returns>
/// <exception cref="System.IO.IOException"/>
internal virtual long SendBlock(DataOutputStream @out, OutputStream baseStream, DataTransferThrottler
throttler)
{
TraceScope scope = Trace.StartSpan("sendBlock_" + block.GetBlockId(), Sampler.Never
);
try
{
return(DoSendBlock(@out, baseStream, throttler));
}
finally
{
scope.Close();
}
}
/// <exception cref="System.IO.IOException"/>
protected override int ReadChunk(long pos, byte[] buf, int offset, int len, byte[]
checksumBuf)
{
lock (this)
{
TraceScope scope = Trace.StartSpan("RemoteBlockReader#readChunk(" + blockId + ")"
, Sampler.Never);
try
{
return(ReadChunkImpl(pos, buf, offset, len, checksumBuf));
}
finally
{
scope.Close();
}
}
}
/// <summary>
/// Receive
/// <see cref="OP.RequestShortCircuitShm"/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void OpRequestShortCircuitShm(DataInputStream @in)
{
DataTransferProtos.ShortCircuitShmRequestProto proto = DataTransferProtos.ShortCircuitShmRequestProto
.ParseFrom(PBHelper.VintPrefixed(@in));
TraceScope traceScope = DataTransferProtoUtil.ContinueTraceSpan(proto.GetTraceInfo
(), proto.GetType().Name);
try
{
RequestShortCircuitShm(proto.GetClientName());
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
}
/// <summary>
/// Receive
/// <see cref="OP.ReleaseShortCircuitFds"/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void OpReleaseShortCircuitFds(DataInputStream @in)
{
DataTransferProtos.ReleaseShortCircuitAccessRequestProto proto = DataTransferProtos.ReleaseShortCircuitAccessRequestProto
.ParseFrom(PBHelper.VintPrefixed(@in));
TraceScope traceScope = DataTransferProtoUtil.ContinueTraceSpan(proto.GetTraceInfo
(), proto.GetType().Name);
try
{
ReleaseShortCircuitFds(PBHelper.Convert(proto.GetSlotId()));
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void WriteWithTracing()
{
long startTime = Runtime.CurrentTimeMillis();
TraceScope ts = Trace.StartSpan("testWriteTraceHooks", Sampler.Always);
WriteTestFile("testWriteTraceHooks.dat");
long endTime = Runtime.CurrentTimeMillis();
ts.Close();
string[] expectedSpanNames = new string[] { "testWriteTraceHooks", "org.apache.hadoop.hdfs.protocol.ClientProtocol.create"
, "ClientNamenodeProtocol#create", "org.apache.hadoop.hdfs.protocol.ClientProtocol.fsync"
, "ClientNamenodeProtocol#fsync", "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete"
, "ClientNamenodeProtocol#complete", "newStreamForCreate", "DFSOutputStream#writeChunk"
, "DFSOutputStream#close", "dataStreamer", "OpWriteBlockProto", "org.apache.hadoop.hdfs.protocol.ClientProtocol.addBlock"
, "ClientNamenodeProtocol#addBlock" };
AssertSpanNamesFound(expectedSpanNames);
// The trace should last about the same amount of time as the test
IDictionary <string, IList <Span> > map = TestTracing.SetSpanReceiver.SetHolder.GetMap
();
Span s = map["testWriteTraceHooks"][0];
NUnit.Framework.Assert.IsNotNull(s);
long spanStart = s.GetStartTimeMillis();
long spanEnd = s.GetStopTimeMillis();
// Spans homed in the top trace shoud have same trace id.
// Spans having multiple parents (e.g. "dataStreamer" added by HDFS-7054)
// and children of them are exception.
string[] spansInTopTrace = new string[] { "testWriteTraceHooks", "org.apache.hadoop.hdfs.protocol.ClientProtocol.create"
, "ClientNamenodeProtocol#create", "org.apache.hadoop.hdfs.protocol.ClientProtocol.fsync"
, "ClientNamenodeProtocol#fsync", "org.apache.hadoop.hdfs.protocol.ClientProtocol.complete"
, "ClientNamenodeProtocol#complete", "newStreamForCreate", "DFSOutputStream#writeChunk"
, "DFSOutputStream#close" };
foreach (string desc in spansInTopTrace)
{
foreach (Span span in map[desc])
{
NUnit.Framework.Assert.AreEqual(ts.GetSpan().GetTraceId(), span.GetTraceId());
}
}
TestTracing.SetSpanReceiver.SetHolder.spans.Clear();
}
/// <summary>Receive OP_BLOCK_CHECKSUM</summary>
/// <exception cref="System.IO.IOException"/>
private void OpBlockChecksum(DataInputStream @in)
{
DataTransferProtos.OpBlockChecksumProto proto = DataTransferProtos.OpBlockChecksumProto
.ParseFrom(PBHelper.VintPrefixed(@in));
TraceScope traceScope = DataTransferProtoUtil.ContinueTraceSpan(proto.GetHeader()
, proto.GetType().Name);
try
{
BlockChecksum(PBHelper.Convert(proto.GetHeader().GetBlock()), PBHelper.Convert(proto
.GetHeader().GetToken()));
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
}
/// <exception cref="System.IO.IOException"/>
public override BatchedRemoteIterator.BatchedEntries <CacheDirectiveEntry> MakeRequest
(long prevKey)
{
BatchedRemoteIterator.BatchedEntries <CacheDirectiveEntry> entries = null;
TraceScope scope = Trace.StartSpan("listCacheDirectives", traceSampler);
try
{
entries = namenode.ListCacheDirectives(prevKey, filter);
}
catch (IOException e)
{
if (e.Message.Contains("Filtering by ID is unsupported"))
{
// Retry case for old servers, do the filtering client-side
long id = filter.GetId();
filter = RemoveIdFromFilter(filter);
// Using id - 1 as prevId should get us a window containing the id
// This is somewhat brittle, since it depends on directives being
// returned in order of ascending ID.
entries = namenode.ListCacheDirectives(id - 1, filter);
for (int i = 0; i < entries.Size(); i++)
{
CacheDirectiveEntry entry = entries.Get(i);
if (entry.GetInfo().GetId().Equals((long)id))
{
return(new CacheDirectiveIterator.SingleEntry(entry));
}
}
throw new RemoteException(typeof(InvalidRequestException).FullName, "Did not find requested id "
+ id);
}
throw;
}
finally
{
scope.Close();
}
Preconditions.CheckNotNull(entries);
return(entries);
}
/// <summary>
/// Returns the next event batch in the stream, waiting up to the specified
/// amount of time for a new batch.
/// </summary>
/// <remarks>
/// Returns the next event batch in the stream, waiting up to the specified
/// amount of time for a new batch. Returns null if one is not available at the
/// end of the specified amount of time. The time before the method returns may
/// exceed the specified amount of time by up to the time required for an RPC
/// to the NameNode.
/// </remarks>
/// <param name="time">number of units of the given TimeUnit to wait</param>
/// <param name="tu">the desired TimeUnit</param>
/// <exception cref="System.IO.IOException">
/// see
/// <see cref="Poll()"/>
/// </exception>
/// <exception cref="Org.Apache.Hadoop.Hdfs.Inotify.MissingEventsException">
/// see
/// <see cref="Poll()"/>
/// </exception>
/// <exception cref="System.Exception">if the calling thread is interrupted</exception>
public virtual EventBatch Poll(long time, TimeUnit tu)
{
TraceScope scope = Trace.StartSpan("inotifyPollWithTimeout", traceSampler);
EventBatch next = null;
try
{
long initialTime = Time.MonotonicNow();
long totalWait = TimeUnit.Milliseconds.Convert(time, tu);
long nextWait = InitialWaitMs;
while ((next = Poll()) == null)
{
long timeLeft = totalWait - (Time.MonotonicNow() - initialTime);
if (timeLeft <= 0)
{
Log.Debug("timed poll(): timed out");
break;
}
else
{
if (timeLeft < nextWait * 2)
{
nextWait = timeLeft;
}
else
{
nextWait *= 2;
}
}
Log.Debug("timed poll(): poll() returned null, sleeping for {} ms", nextWait);
Sharpen.Thread.Sleep(nextWait);
}
}
finally
{
scope.Close();
}
return(next);
}
/// <exception cref="System.IO.IOException"/>
public virtual int Read(byte[] buf, int off, int len)
{
lock (this)
{
UUID randomId = null;
if (Log.IsTraceEnabled())
{
randomId = UUID.RandomUUID();
Log.Trace(string.Format("Starting read #%s file %s from datanode %s", randomId.ToString
(), this.filename, this.datanodeID.GetHostName()));
}
if (curDataSlice == null || curDataSlice.Remaining() == 0 && bytesNeededToFinish
> 0)
{
TraceScope scope = Trace.StartSpan("RemoteBlockReader2#readNextPacket(" + blockId
+ ")", Sampler.Never);
try
{
ReadNextPacket();
}
finally
{
scope.Close();
}
}
if (Log.IsTraceEnabled())
{
Log.Trace(string.Format("Finishing read #" + randomId));
}
if (curDataSlice.Remaining() == 0)
{
// we're at EOF now
return(-1);
}
int nRead = Math.Min(curDataSlice.Remaining(), len);
curDataSlice.Get(buf, off, nRead);
return(nRead);
}
}
/// <summary>
/// Receive
/// <see cref="OP.RequestShortCircuitFds"/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void OpRequestShortCircuitFds(DataInputStream @in)
{
DataTransferProtos.OpRequestShortCircuitAccessProto proto = DataTransferProtos.OpRequestShortCircuitAccessProto
.ParseFrom(PBHelper.VintPrefixed(@in));
ShortCircuitShm.SlotId slotId = (proto.HasSlotId()) ? PBHelper.Convert(proto.GetSlotId
()) : null;
TraceScope traceScope = DataTransferProtoUtil.ContinueTraceSpan(proto.GetHeader()
, proto.GetType().Name);
try
{
RequestShortCircuitFds(PBHelper.Convert(proto.GetHeader().GetBlock()), PBHelper.Convert
(proto.GetHeader().GetToken()), slotId, proto.GetMaxVersion(), proto.GetSupportsReceiptVerification
());
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
}
/// <summary>
/// Receive
/// <see cref="OP.TransferBlock"/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
private void OpTransferBlock(DataInputStream @in)
{
DataTransferProtos.OpTransferBlockProto proto = DataTransferProtos.OpTransferBlockProto
.ParseFrom(PBHelper.VintPrefixed(@in));
DatanodeInfo[] targets = PBHelper.Convert(proto.GetTargetsList());
TraceScope traceScope = DataTransferProtoUtil.ContinueTraceSpan(proto.GetHeader()
, proto.GetType().Name);
try
{
TransferBlock(PBHelper.Convert(proto.GetHeader().GetBaseHeader().GetBlock()), PBHelper
.Convert(proto.GetHeader().GetBaseHeader().GetToken()), proto.GetHeader().GetClientName
(), targets, PBHelper.ConvertStorageTypes(proto.GetTargetStorageTypesList(), targets
.Length));
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
}
/// <summary>Receive OP_READ_BLOCK</summary>
/// <exception cref="System.IO.IOException"/>
private void OpReadBlock()
{
DataTransferProtos.OpReadBlockProto proto = DataTransferProtos.OpReadBlockProto.ParseFrom
(PBHelper.VintPrefixed(@in));
TraceScope traceScope = DataTransferProtoUtil.ContinueTraceSpan(proto.GetHeader()
, proto.GetType().Name);
try
{
ReadBlock(PBHelper.Convert(proto.GetHeader().GetBaseHeader().GetBlock()), PBHelper
.Convert(proto.GetHeader().GetBaseHeader().GetToken()), proto.GetHeader().GetClientName
(), proto.GetOffset(), proto.GetLen(), proto.GetSendChecksums(), (proto.HasCachingStrategy
() ? GetCachingStrategy(proto.GetCachingStrategy()) : CachingStrategy.NewDefaultStrategy
()));
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
}
/// <summary>Read from the block file into a buffer.</summary>
/// <remarks>
/// Read from the block file into a buffer.
/// This function overwrites checksumBuf. It will increment dataPos.
/// </remarks>
/// <param name="buf">
/// The buffer to read into. May be dataBuf.
/// The position and limit of this buffer should be set to
/// multiples of the checksum size.
/// </param>
/// <param name="canSkipChecksum">True if we can skip checksumming.</param>
/// <returns>Total bytes read. 0 on EOF.</returns>
/// <exception cref="System.IO.IOException"/>
private int FillBuffer(ByteBuffer buf, bool canSkipChecksum)
{
lock (this)
{
TraceScope scope = Trace.StartSpan("BlockReaderLocal#fillBuffer(" + block.GetBlockId
() + ")", Sampler.Never);
try
{
int total = 0;
long startDataPos = dataPos;
int startBufPos = buf.Position();
while (buf.HasRemaining())
{
int nRead = dataIn.Read(buf, dataPos);
if (nRead < 0)
{
break;
}
dataPos += nRead;
total += nRead;
}
if (canSkipChecksum)
{
FreeChecksumBufIfExists();
return(total);
}
if (total > 0)
{
try
{
buf.Limit(buf.Position());
buf.Position(startBufPos);
CreateChecksumBufIfNeeded();
int checksumsNeeded = (total + bytesPerChecksum - 1) / bytesPerChecksum;
checksumBuf.Clear();
checksumBuf.Limit(checksumsNeeded * checksumSize);
long checksumPos = BlockMetadataHeader.GetHeaderSize() + ((startDataPos / bytesPerChecksum
) * checksumSize);
while (checksumBuf.HasRemaining())
{
int nRead = checksumIn.Read(checksumBuf, checksumPos);
if (nRead < 0)
{
throw new IOException("Got unexpected checksum file EOF at " + checksumPos + ", block file position "
+ startDataPos + " for " + "block " + block + " of file " + filename);
}
checksumPos += nRead;
}
checksumBuf.Flip();
checksum.VerifyChunkedSums(buf, checksumBuf, filename, startDataPos);
}
finally
{
buf.Position(buf.Limit());
}
}
return(total);
}
finally
{
scope.Close();
}
}
}
/// <summary>This is the client side invoker of RPC method.</summary>
/// <remarks>
/// This is the client side invoker of RPC method. It only throws
/// ServiceException, since the invocation proxy expects only
/// ServiceException to be thrown by the method in case protobuf service.
/// ServiceException has the following causes:
/// <ol>
/// <li>Exceptions encountered on the client side in this method are
/// set as cause in ServiceException as is.</li>
/// <li>Exceptions from the server are wrapped in RemoteException and are
/// set as cause in ServiceException</li>
/// </ol>
/// Note that the client calling protobuf RPC methods, must handle
/// ServiceException by getting the cause from the ServiceException. If the
/// cause is RemoteException, then unwrap it to get the exception thrown by
/// the server.
/// </remarks>
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual object Invoke(object proxy, MethodInfo method, object[] args)
{
long startTime = 0;
if (Log.IsDebugEnabled())
{
startTime = Time.Now();
}
if (args.Length != 2)
{
// RpcController + Message
throw new ServiceException("Too many parameters for request. Method: [" + method.
Name + "]" + ", Expected: 2, Actual: " + args.Length);
}
if (args[1] == null)
{
throw new ServiceException("null param while calling Method: [" + method.Name + "]"
);
}
TraceScope traceScope = null;
// if Tracing is on then start a new span for this rpc.
// guard it in the if statement to make sure there isn't
// any extra string manipulation.
if (Trace.IsTracing())
{
traceScope = Trace.StartSpan(RpcClientUtil.MethodToTraceString(method));
}
ProtobufRpcEngineProtos.RequestHeaderProto rpcRequestHeader = ConstructRpcRequestHeader
(method);
if (Log.IsTraceEnabled())
{
Log.Trace(Thread.CurrentThread().GetId() + ": Call -> " + remoteId + ": "
+ method.Name + " {" + TextFormat.ShortDebugString((Message)args[1]) + "}");
}
Message theRequest = (Message)args[1];
ProtobufRpcEngine.RpcResponseWrapper val;
try
{
val = (ProtobufRpcEngine.RpcResponseWrapper)client.Call(RPC.RpcKind.RpcProtocolBuffer
, new ProtobufRpcEngine.RpcRequestWrapper(rpcRequestHeader, theRequest), remoteId
, fallbackToSimpleAuth);
}
catch (Exception e)
{
if (Log.IsTraceEnabled())
{
Log.Trace(Thread.CurrentThread().GetId() + ": Exception <- " + remoteId +
": " + method.Name + " {" + e + "}");
}
if (Trace.IsTracing())
{
traceScope.GetSpan().AddTimelineAnnotation("Call got exception: " + e.Message);
}
throw new ServiceException(e);
}
finally
{
if (traceScope != null)
{
traceScope.Close();
}
}
if (Log.IsDebugEnabled())
{
long callTime = Time.Now() - startTime;
Log.Debug("Call: " + method.Name + " took " + callTime + "ms");
}
Message prototype = null;
try
{
prototype = GetReturnProtoType(method);
}
catch (Exception e)
{
throw new ServiceException(e);
}
Message returnMessage;
try
{
returnMessage = prototype.NewBuilderForType().MergeFrom(val.theResponseRead).Build
();
if (Log.IsTraceEnabled())
{
Log.Trace(Thread.CurrentThread().GetId() + ": Response <- " + remoteId +
": " + method.Name + " {" + TextFormat.ShortDebugString(returnMessage) + "}");
}
}
catch (Exception e)
{
throw new ServiceException(e);
}
return(returnMessage);
}