Exemplo n.º 1
0
        /// <returns>
        /// a list of
        /// <see cref="Org.Apache.Hadoop.FS.StorageType"/>
        /// s for storing the replicas of a block.
        /// </returns>
        public virtual IList <StorageType> ChooseStorageTypes(short replication)
        {
            IList <StorageType> types = new List <StorageType>();
            int i = 0;
            int j = 0;

            // Do not return transient storage types. We will not have accurate
            // usage information for transient types.
            for (; i < replication && j < storageTypes.Length; ++j)
            {
                if (!storageTypes[j].IsTransient())
                {
                    types.AddItem(storageTypes[j]);
                    ++i;
                }
            }
            StorageType last = storageTypes[storageTypes.Length - 1];

            if (!last.IsTransient())
            {
                for (; i < replication; i++)
                {
                    types.AddItem(last);
                }
            }
            return(types);
        }
Exemplo n.º 2
0
 private bool CreateNoChecksumContext()
 {
     if (verifyChecksum)
     {
         if (storageType != null && storageType.IsTransient())
         {
             // Checksums are not stored for replicas on transient storage.  We do not
             // anchor, because we do not intend for client activity to block eviction
             // from transient storage on the DataNode side.
             return(true);
         }
         else
         {
             return(replica.AddNoChecksumAnchor());
         }
     }
     else
     {
         return(true);
     }
 }
Exemplo n.º 3
0
        protected internal virtual ThreadPoolExecutor InitializeCacheExecutor(FilePath parent
                                                                              )
        {
            if (storageType.IsTransient())
            {
                return(null);
            }
            if (dataset.datanode == null)
            {
                // FsVolumeImpl is used in test.
                return(null);
            }
            int maxNumThreads = dataset.datanode.GetConf().GetInt(DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeKey
                                                                  , DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeDefault);
            ThreadFactory workerFactory = new ThreadFactoryBuilder().SetDaemon(true).SetNameFormat
                                              ("FsVolumeImplWorker-" + parent.ToString() + "-%d").Build();
            ThreadPoolExecutor executor = new ThreadPoolExecutor(1, maxNumThreads, 60, TimeUnit
                                                                 .Seconds, new LinkedBlockingQueue <Runnable>(), workerFactory);

            executor.AllowCoreThreadTimeOut(true);
            return(executor);
        }
Exemplo n.º 4
0
        /// <exception cref="System.IO.IOException"/>
        private static BlockLocalPathInfo GetBlockPathInfo(UserGroupInformation ugi, ExtendedBlock
                                                           blk, DatanodeInfo node, Configuration conf, int timeout, Org.Apache.Hadoop.Security.Token.Token
                                                           <BlockTokenIdentifier> token, bool connectToDnViaHostname, StorageType storageType
                                                           )
        {
            BlockReaderLocalLegacy.LocalDatanodeInfo localDatanodeInfo = GetLocalDatanodeInfo
                                                                             (node.GetIpcPort());
            BlockLocalPathInfo     pathinfo = null;
            ClientDatanodeProtocol proxy    = localDatanodeInfo.GetDatanodeProxy(ugi, node, conf
                                                                                 , timeout, connectToDnViaHostname);

            try
            {
                // make RPC to local datanode to find local pathnames of blocks
                pathinfo = proxy.GetBlockLocalPathInfo(blk, token);
                // We cannot cache the path information for a replica on transient storage.
                // If the replica gets evicted, then it moves to a different path.  Then,
                // our next attempt to read from the cached path would fail to find the
                // file.  Additionally, the failure would cause us to disable legacy
                // short-circuit read for all subsequent use in the ClientContext.  Unlike
                // the newer short-circuit read implementation, we have no communication
                // channel for the DataNode to notify the client that the path has been
                // invalidated.  Therefore, our only option is to skip caching.
                if (pathinfo != null && !storageType.IsTransient())
                {
                    if (Log.IsDebugEnabled())
                    {
                        Log.Debug("Cached location of block " + blk + " as " + pathinfo);
                    }
                    localDatanodeInfo.SetBlockLocalPathInfo(blk, pathinfo);
                }
            }
            catch (IOException e)
            {
                localDatanodeInfo.ResetDatanodeProxy();
                // Reset proxy on error
                throw;
            }
            return(pathinfo);
        }
Exemplo n.º 5
0
        // Multiple datanodes could be running on the local machine. Store proxies in
        // a map keyed by the ipc port of the datanode.
        // reader for the data file
        // reader for the checksum file
        /// <summary>The only way this object can be instantiated.</summary>
        /// <exception cref="System.IO.IOException"/>
        internal static BlockReaderLocalLegacy NewBlockReader(DFSClient.Conf conf, UserGroupInformation
                                                              userGroupInformation, Configuration configuration, string file, ExtendedBlock blk
                                                              , Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, DatanodeInfo
                                                              node, long startOffset, long length, StorageType storageType)
        {
            BlockReaderLocalLegacy.LocalDatanodeInfo localDatanodeInfo = GetLocalDatanodeInfo
                                                                             (node.GetIpcPort());
            // check the cache first
            BlockLocalPathInfo pathinfo = localDatanodeInfo.GetBlockLocalPathInfo(blk);

            if (pathinfo == null)
            {
                if (userGroupInformation == null)
                {
                    userGroupInformation = UserGroupInformation.GetCurrentUser();
                }
                pathinfo = GetBlockPathInfo(userGroupInformation, blk, node, configuration, conf.
                                            socketTimeout, token, conf.connectToDnViaHostname, storageType);
            }
            // check to see if the file exists. It may so happen that the
            // HDFS file has been deleted and this block-lookup is occurring
            // on behalf of a new HDFS file. This time, the block file could
            // be residing in a different portion of the fs.data.dir directory.
            // In this case, we remove this entry from the cache. The next
            // call to this method will re-populate the cache.
            FileInputStream        dataIn           = null;
            FileInputStream        checksumIn       = null;
            BlockReaderLocalLegacy localBlockReader = null;
            bool skipChecksumCheck = conf.skipShortCircuitChecksums || storageType.IsTransient
                                         ();

            try
            {
                // get a local file system
                FilePath blkfile = new FilePath(pathinfo.GetBlockPath());
                dataIn = new FileInputStream(blkfile);
                if (Log.IsDebugEnabled())
                {
                    Log.Debug("New BlockReaderLocalLegacy for file " + blkfile + " of size " + blkfile
                              .Length() + " startOffset " + startOffset + " length " + length + " short circuit checksum "
                              + !skipChecksumCheck);
                }
                if (!skipChecksumCheck)
                {
                    // get the metadata file
                    FilePath metafile = new FilePath(pathinfo.GetMetaPath());
                    checksumIn = new FileInputStream(metafile);
                    DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(new DataInputStream(
                                                                                     checksumIn), blk);
                    long firstChunkOffset = startOffset - (startOffset % checksum.GetBytesPerChecksum
                                                               ());
                    localBlockReader = new BlockReaderLocalLegacy(conf, file, blk, token, startOffset
                                                                  , length, pathinfo, checksum, true, dataIn, firstChunkOffset, checksumIn);
                }
                else
                {
                    localBlockReader = new BlockReaderLocalLegacy(conf, file, blk, token, startOffset
                                                                  , length, pathinfo, dataIn);
                }
            }
            catch (IOException e)
            {
                // remove from cache
                localDatanodeInfo.RemoveBlockLocalPathInfo(blk);
                DFSClient.Log.Warn("BlockReaderLocalLegacy: Removing " + blk + " from cache because local file "
                                   + pathinfo.GetBlockPath() + " could not be opened.");
                throw;
            }
            finally
            {
                if (localBlockReader == null)
                {
                    if (dataIn != null)
                    {
                        dataIn.Close();
                    }
                    if (checksumIn != null)
                    {
                        checksumIn.Close();
                    }
                }
            }
            return(localBlockReader);
        }