public LineManagement() { nowLine = new LineFileBlockStruct(); supportingLines = new HashSet <string>(); subwayReader = new SeekReader <SubwayFileBlockStruct>(); haveLineReader = new BlockReader <HaveLineFileBlockStruct>(); lineReader = new SeekReader <LineFileBlockStruct>(); nowSubwayStop = new HashSet <SubwayFileBlockStruct>(); }
public void CanReadFullBlock() { var bl = new BlockReader(new MemOwner(CommonData.SingleBlock)); while (bl.TryGetNextKey(out var key)) { var mem = bl.GetCurrentValue(); } }
// try reading a block using a BlockReader directly private static void TryRead(Configuration conf, LocatedBlock lblock, bool shouldSucceed ) { IPEndPoint targetAddr = null; IOException ioe = null; BlockReader blockReader = null; ExtendedBlock block = lblock.GetBlock(); try { DatanodeInfo[] nodes = lblock.GetLocations(); targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr()); blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).SetFileName(BlockReaderFactory .GetFileName(targetAddr, "test-blockpoolid", block.GetBlockId())).SetBlock(block ).SetBlockToken(lblock.GetBlockToken()).SetInetSocketAddress(targetAddr).SetStartOffset (0).SetLength(-1).SetVerifyChecksum(true).SetClientName("TestBlockTokenWithDFS") .SetDatanodeInfo(nodes[0]).SetCachingStrategy(CachingStrategy.NewDefaultStrategy ()).SetClientCacheContext(ClientContext.GetFromConf(conf)).SetConfiguration(conf ).SetRemotePeerFactory(new _RemotePeerFactory_162(conf)).Build(); } catch (IOException ex) { ioe = ex; } finally { if (blockReader != null) { try { blockReader.Close(); } catch (IOException e) { throw new RuntimeException(e); } } } if (shouldSucceed) { NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is invalid, " + "when it is expected to be valid" , blockReader); } else { NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is valid, " + "when it is expected to be invalid" , ioe); NUnit.Framework.Assert.IsTrue("OP_READ_BLOCK failed due to reasons other than access token: " , ioe is InvalidBlockTokenException); } }
/// <summary>Build a BlockReader with the given options.</summary> /// <remarks> /// Build a BlockReader with the given options. /// This function will do the best it can to create a block reader that meets /// all of our requirements. We prefer short-circuit block readers /// (BlockReaderLocal and BlockReaderLocalLegacy) over remote ones, since the /// former avoid the overhead of socket communication. If short-circuit is /// unavailable, our next fallback is data transfer over UNIX domain sockets, /// if dfs.client.domain.socket.data.traffic has been enabled. If that doesn't /// work, we will try to create a remote block reader that operates over TCP /// sockets. /// There are a few caches that are important here. /// The ShortCircuitCache stores file descriptor objects which have been passed /// from the DataNode. /// The DomainSocketFactory stores information about UNIX domain socket paths /// that we not been able to use in the past, so that we don't waste time /// retrying them over and over. (Like all the caches, it does have a timeout, /// though.) /// The PeerCache stores peers that we have used in the past. If we can reuse /// one of these peers, we avoid the overhead of re-opening a socket. However, /// if the socket has been timed out on the remote end, our attempt to reuse /// the socket may end with an IOException. For that reason, we limit our /// attempts at socket reuse to dfs.client.cached.conn.retry times. After /// that, we create new sockets. This avoids the problem where a thread tries /// to talk to a peer that it hasn't talked to in a while, and has to clean out /// every entry in a socket cache full of stale entries. /// </remarks> /// <returns>The new BlockReader. We will not return null.</returns> /// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"> /// If the block token was invalid. /// InvalidEncryptionKeyException /// If the encryption key was invalid. /// Other IOException /// If there was another problem. /// </exception> /// <exception cref="System.IO.IOException"/> public virtual BlockReader Build() { BlockReader reader = null; Preconditions.CheckNotNull(configuration); if (conf.shortCircuitLocalReads && allowShortCircuitLocalReads) { if (clientContext.GetUseLegacyBlockReaderLocal()) { reader = GetLegacyBlockReaderLocal(); if (reader != null) { if (Log.IsTraceEnabled()) { Log.Trace(this + ": returning new legacy block reader local."); } return(reader); } } else { reader = GetBlockReaderLocal(); if (reader != null) { if (Log.IsTraceEnabled()) { Log.Trace(this + ": returning new block reader local."); } return(reader); } } } if (conf.domainSocketDataTraffic) { reader = GetRemoteBlockReaderFromDomain(); if (reader != null) { if (Log.IsTraceEnabled()) { Log.Trace(this + ": returning new remote block reader using " + "UNIX domain socket on " + pathInfo.GetPath()); } return(reader); } } Preconditions.CheckState(!DFSInputStream.tcpReadsDisabledForTesting, "TCP reads were disabled for testing, but we failed to " + "do a non-TCP read."); return(GetRemoteBlockReaderFromTcp()); }
public (StringBuilder, StringReader) Execute( string currentLine, StringReader templateReader, StringBuilder output, IDictionary <string, object> model = null) { var blockBody = BlockReader.ReadBlock(new StringReader(currentLine)); var engine = new RazorAlikeEngine(); var resolved = engine.Run(blockBody.ReadToEnd(), model); output.AppendLine(resolved); return(output, templateReader); }
public BuildBuffer(Stream stream, int fallbackCodePage) { FileStream fStrm = (stream as FileStream); if (fStrm != null) FileName = fStrm.Name; NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage); }
public BuildBuffer(Stream stream) { FileStream fStrm = (stream as FileStream); if (fStrm != null) FileName = fStrm.Name; NextBlk = BlockReaderFactory.Raw(stream); }