public LineManagement()
{
nowLine = new LineFileBlockStruct();
supportingLines = new HashSet <string>();
subwayReader = new SeekReader <SubwayFileBlockStruct>();
haveLineReader = new BlockReader <HaveLineFileBlockStruct>();
lineReader = new SeekReader <LineFileBlockStruct>();
nowSubwayStop = new HashSet <SubwayFileBlockStruct>();
}
public void CanReadFullBlock()
{
var bl = new BlockReader(new MemOwner(CommonData.SingleBlock));
while (bl.TryGetNextKey(out var key))
{
var mem = bl.GetCurrentValue();
}
}
// try reading a block using a BlockReader directly
private static void TryRead(Configuration conf, LocatedBlock lblock, bool shouldSucceed
)
{
IPEndPoint targetAddr = null;
IOException ioe = null;
BlockReader blockReader = null;
ExtendedBlock block = lblock.GetBlock();
try
{
DatanodeInfo[] nodes = lblock.GetLocations();
targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr());
blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).SetFileName(BlockReaderFactory
.GetFileName(targetAddr, "test-blockpoolid", block.GetBlockId())).SetBlock(block
).SetBlockToken(lblock.GetBlockToken()).SetInetSocketAddress(targetAddr).SetStartOffset
(0).SetLength(-1).SetVerifyChecksum(true).SetClientName("TestBlockTokenWithDFS")
.SetDatanodeInfo(nodes[0]).SetCachingStrategy(CachingStrategy.NewDefaultStrategy
()).SetClientCacheContext(ClientContext.GetFromConf(conf)).SetConfiguration(conf
).SetRemotePeerFactory(new _RemotePeerFactory_162(conf)).Build();
}
catch (IOException ex)
{
ioe = ex;
}
finally
{
if (blockReader != null)
{
try
{
blockReader.Close();
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}
if (shouldSucceed)
{
NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is invalid, " + "when it is expected to be valid"
, blockReader);
}
else
{
NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is valid, " + "when it is expected to be invalid"
, ioe);
NUnit.Framework.Assert.IsTrue("OP_READ_BLOCK failed due to reasons other than access token: "
, ioe is InvalidBlockTokenException);
}
}
/// <summary>Build a BlockReader with the given options.</summary>
/// <remarks>
/// Build a BlockReader with the given options.
/// This function will do the best it can to create a block reader that meets
/// all of our requirements. We prefer short-circuit block readers
/// (BlockReaderLocal and BlockReaderLocalLegacy) over remote ones, since the
/// former avoid the overhead of socket communication. If short-circuit is
/// unavailable, our next fallback is data transfer over UNIX domain sockets,
/// if dfs.client.domain.socket.data.traffic has been enabled. If that doesn't
/// work, we will try to create a remote block reader that operates over TCP
/// sockets.
/// There are a few caches that are important here.
/// The ShortCircuitCache stores file descriptor objects which have been passed
/// from the DataNode.
/// The DomainSocketFactory stores information about UNIX domain socket paths
/// that we not been able to use in the past, so that we don't waste time
/// retrying them over and over. (Like all the caches, it does have a timeout,
/// though.)
/// The PeerCache stores peers that we have used in the past. If we can reuse
/// one of these peers, we avoid the overhead of re-opening a socket. However,
/// if the socket has been timed out on the remote end, our attempt to reuse
/// the socket may end with an IOException. For that reason, we limit our
/// attempts at socket reuse to dfs.client.cached.conn.retry times. After
/// that, we create new sockets. This avoids the problem where a thread tries
/// to talk to a peer that it hasn't talked to in a while, and has to clean out
/// every entry in a socket cache full of stale entries.
/// </remarks>
/// <returns>The new BlockReader. We will not return null.</returns>
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken">
/// If the block token was invalid.
/// InvalidEncryptionKeyException
/// If the encryption key was invalid.
/// Other IOException
/// If there was another problem.
/// </exception>
/// <exception cref="System.IO.IOException"/>
public virtual BlockReader Build()
{
BlockReader reader = null;
Preconditions.CheckNotNull(configuration);
if (conf.shortCircuitLocalReads && allowShortCircuitLocalReads)
{
if (clientContext.GetUseLegacyBlockReaderLocal())
{
reader = GetLegacyBlockReaderLocal();
if (reader != null)
{
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": returning new legacy block reader local.");
}
return(reader);
}
}
else
{
reader = GetBlockReaderLocal();
if (reader != null)
{
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": returning new block reader local.");
}
return(reader);
}
}
}
if (conf.domainSocketDataTraffic)
{
reader = GetRemoteBlockReaderFromDomain();
if (reader != null)
{
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": returning new remote block reader using " + "UNIX domain socket on "
+ pathInfo.GetPath());
}
return(reader);
}
}
Preconditions.CheckState(!DFSInputStream.tcpReadsDisabledForTesting, "TCP reads were disabled for testing, but we failed to "
+ "do a non-TCP read.");
return(GetRemoteBlockReaderFromTcp());
}
public (StringBuilder, StringReader) Execute(
string currentLine,
StringReader templateReader,
StringBuilder output,
IDictionary <string, object> model = null)
{
var blockBody = BlockReader.ReadBlock(new StringReader(currentLine));
var engine = new RazorAlikeEngine();
var resolved = engine.Run(blockBody.ReadToEnd(), model);
output.AppendLine(resolved);
return(output, templateReader);
}
public BuildBuffer(Stream stream, int fallbackCodePage)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
}
public BuildBuffer(Stream stream)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Raw(stream);
}
