Exemple #1
0
        /// <summary>
        /// Queries datanodes for the blocks specified in <code>datanodeBlocks</code>,
        /// making one RPC to each datanode.
        /// </summary>
        /// <remarks>
        /// Queries datanodes for the blocks specified in <code>datanodeBlocks</code>,
        /// making one RPC to each datanode. These RPCs are made in parallel using a
        /// threadpool.
        /// </remarks>
        /// <param name="datanodeBlocks">Map of datanodes to the blocks present on the DN</param>
        /// <returns>metadatas Map of datanodes to block metadata of the DN</returns>
        /// <exception cref="Org.Apache.Hadoop.Hdfs.Security.Token.Block.InvalidBlockTokenException
        ///     ">if client does not have read access on a requested block</exception>
        internal static IDictionary <DatanodeInfo, HdfsBlocksMetadata> QueryDatanodesForHdfsBlocksMetadata
            (Configuration conf, IDictionary <DatanodeInfo, IList <LocatedBlock> > datanodeBlocks
            , int poolsize, int timeoutMs, bool connectToDnViaHostname)
        {
            IList <BlockStorageLocationUtil.VolumeBlockLocationCallable> callables = CreateVolumeBlockLocationCallables
                                                                                         (conf, datanodeBlocks, timeoutMs, connectToDnViaHostname, Trace.CurrentSpan());
            // Use a thread pool to execute the Callables in parallel
            IList <Future <HdfsBlocksMetadata> > futures = new AList <Future <HdfsBlocksMetadata> >
                                                               ();
            ExecutorService executor = new ScheduledThreadPoolExecutor(poolsize);

            try
            {
                futures = executor.InvokeAll(callables, timeoutMs, TimeUnit.Milliseconds);
            }
            catch (Exception)
            {
            }
            // Swallow the exception here, because we can return partial results
            executor.Shutdown();
            IDictionary <DatanodeInfo, HdfsBlocksMetadata> metadatas = Maps.NewHashMapWithExpectedSize
                                                                           (datanodeBlocks.Count);

            // Fill in metadatas with results from DN RPCs, where possible
            for (int i = 0; i < futures.Count; i++)
            {
                BlockStorageLocationUtil.VolumeBlockLocationCallable callable = callables[i];
                DatanodeInfo datanode = callable.GetDatanodeInfo();
                Future <HdfsBlocksMetadata> future = futures[i];
                try
                {
                    HdfsBlocksMetadata metadata = future.Get();
                    metadatas[callable.GetDatanodeInfo()] = metadata;
                }
                catch (CancellationException e)
                {
                    Log.Info("Cancelled while waiting for datanode " + datanode.GetIpcAddr(false) + ": "
                             + e.ToString());
                }
                catch (ExecutionException e)
                {
                    Exception t = e.InnerException;
                    if (t is InvalidBlockTokenException)
                    {
                        Log.Warn("Invalid access token when trying to retrieve " + "information from datanode "
                                 + datanode.GetIpcAddr(false));
                        throw (InvalidBlockTokenException)t;
                    }
                    else
                    {
                        if (t is NotSupportedException)
                        {
                            Log.Info("Datanode " + datanode.GetIpcAddr(false) + " does not support" + " required #getHdfsBlocksMetadata() API"
                                     );
                            throw (NotSupportedException)t;
                        }
                        else
                        {
                            Log.Info("Failed to query block locations on datanode " + datanode.GetIpcAddr(false
                                                                                                          ) + ": " + t);
                        }
                    }
                    if (Log.IsDebugEnabled())
                    {
                        Log.Debug("Could not fetch information from datanode", t);
                    }
                }
                catch (Exception)
                {
                    // Shouldn't happen, because invokeAll waits for all Futures to be ready
                    Log.Info("Interrupted while fetching HdfsBlocksMetadata");
                }
            }
            return(metadatas);
        }